@@ -2801,8 +2812,8 @@
{% set s = scores.get(eval_id) %}
{{ "%.2f" | format(s) if s is not none else "—" }}
{% endfor %}
-
{{ trial.steps or 0 }}
-
{{ ((tokens.prompt or 0) + (tokens.completion or 0)) }}
+
{{ trial.steps if trial.steps is number else "—" }}
+
{% if tokens.prompt is number and tokens.completion is number %}{{ tokens.prompt + tokens.completion }}{% else %}—{% endif %}
{% if trial.warnings %}{{ trial.warnings | length }} {% else %}—{% endif %}
{% endfor %}
@@ -3678,13 +3689,27 @@
}
function tokenTotalsByAgent() {
var totals = {};
+ var trialCounts = {};
(tier3Data.trials || []).forEach(function(trial){
var name = trial.agent || "default";
- if (!totals[name]) totals[name] = {prompt:0, completion:0, cached:0};
+ trialCounts[name] = (trialCounts[name] || 0) + 1;
var tokens = trial.tokens || {};
- totals[name].prompt += Number(tokens.prompt || 0);
- totals[name].completion += Number(tokens.completion || 0);
- totals[name].cached += Number(tokens.cached || 0);
+ var prompt = tokens.prompt;
+ var completion = tokens.completion;
+ if (typeof prompt !== "number" || !Number.isFinite(prompt) || prompt < 0 ||
+ typeof completion !== "number" || !Number.isFinite(completion) || completion < 0) return;
+ if (!totals[name]) totals[name] = {prompt:0, completion:0, cached:0, observed:0, cachedComplete:true};
+ totals[name].prompt += prompt;
+ totals[name].completion += completion;
+ totals[name].observed += 1;
+ if (typeof tokens.cached !== "number" || !Number.isFinite(tokens.cached) || tokens.cached < 0) {
+ totals[name].cachedComplete = false;
+ } else {
+ totals[name].cached += tokens.cached;
+ }
+ });
+ Object.keys(totals).forEach(function(name){
+ if (totals[name].observed !== trialCounts[name]) delete totals[name];
});
return totals;
}
@@ -3703,8 +3728,12 @@
label: name,
data: entries.map(function(entry){
var matching = (tier3Data.trials || []).filter(function(t){ return (t.agent || "default") === name && (t.entry_id || t.trial_id || "?") === entry; });
- if (!matching.length) return 0;
- return matching.reduce(function(sum, t){ return sum + Number(t.steps || 0); }, 0) / matching.length;
+ if (!matching.length) return null;
+ var observed = matching.map(function(t){ return t.steps; }).filter(function(value){
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
+ });
+ if (observed.length !== matching.length) return null;
+ return observed.reduce(function(sum, value){ return sum + value; }, 0) / observed.length;
}),
backgroundColor: palette[idx % palette.length] + "cc",
borderRadius: 4
@@ -3773,14 +3802,14 @@
data: {
labels: tokenAgents,
datasets: [
- { label: "Prompt", data: tokenAgents.map(function(a){ return tokenTotals[a].prompt; }), backgroundColor: "#3b82f6cc", borderRadius: 3 },
+ { label: "Prompt (total)", data: tokenAgents.map(function(a){ return tokenTotals[a].prompt; }), backgroundColor: "#3b82f6cc", borderRadius: 3 },
{ label: "Completion", data: tokenAgents.map(function(a){ return tokenTotals[a].completion; }), backgroundColor: "#22c55ecc", borderRadius: 3 },
- { label: "Cached", data: tokenAgents.map(function(a){ return tokenTotals[a].cached; }), backgroundColor: "#8b5cf6cc", borderRadius: 3 }
+ { label: "Cached (included in prompt)", data: tokenAgents.map(function(a){ return tokenTotals[a].cachedComplete ? tokenTotals[a].cached : null; }), backgroundColor: "#8b5cf6cc", borderRadius: 3 }
]
},
options: {
responsive: true,
- scales: { x: { stacked: true, ticks: { color: colors.tick }, grid: { color: colors.grid } }, y: { stacked: true, ticks: { color: colors.label }, grid: { display: false } } },
+ scales: { x: { ticks: { color: colors.tick }, grid: { color: colors.grid } }, y: { beginAtZero: true, ticks: { color: colors.label }, grid: { display: false } } },
plugins: { legend: { labels: { color: colors.label } } }
}
}));
diff --git a/src/skillevaluator/tier3/commands.py b/src/skillevaluator/tier3/commands.py
index 217dfe46..acaa0a74 100644
--- a/src/skillevaluator/tier3/commands.py
+++ b/src/skillevaluator/tier3/commands.py
@@ -599,6 +599,7 @@ def evaluate(
*,
agents: str | None,
env_mode: str,
+ environment_kwarg: tuple[str, ...] = (),
skip_baseline: bool,
n_attempts: int | None,
pass_threshold: float | None,
@@ -653,6 +654,9 @@ def evaluate(
raise ValueError(f"A public LLM provider is required for live evaluation: {exc}") from exc
agent_models = parse_agent_model_overrides(agent_model)
+ from skillevaluator.tier3.evals_config import parse_environment_kwarg_overrides
+
+ environment_kwargs = parse_environment_kwarg_overrides(environment_kwarg, env_mode=env_mode)
unknown_model_agents = sorted(set(agent_models) - set(agent_list))
if unknown_model_agents:
raise ValueError(
@@ -682,6 +686,7 @@ def evaluate(
agent_runtime_preflight=agent_runtime_preflight,
env_mode=env_mode,
env_mode_source="CLI",
+ environment_kwargs=environment_kwargs,
timeout_multiplier=timeout_multiplier,
override_cpus=override_cpus,
override_memory_mb=override_memory_mb,
@@ -700,6 +705,7 @@ def doctor(
*,
agents: str | None,
env_mode: str,
+ environment_kwarg: tuple[str, ...] = (),
verify_models: bool = False,
agent_model: tuple[str, ...] = (),
) -> int:
@@ -775,7 +781,21 @@ def doctor(
else:
rows.append(("Harbor agents", "pass", ", ".join(agent_list)))
- prereq_errors = _check_prerequisites(env_mode=env_mode, agents=agent_list)
+ from skillevaluator.tier3.evals_config import parse_environment_kwarg_overrides
+
+ try:
+ environment_kwargs = parse_environment_kwarg_overrides(environment_kwarg, env_mode=env_mode)
+ except ValueError as exc:
+ environment_kwargs = {}
+ prereq_errors = [str(exc)]
+ else:
+ prerequisite_subprocess_env = dict(next(iter(runtime_plans.values())).subprocess_env) if runtime_plans else None
+ prereq_errors = _check_prerequisites(
+ env_mode=env_mode,
+ agents=agent_list,
+ environment_kwargs=environment_kwargs,
+ subprocess_env=prerequisite_subprocess_env,
+ )
if prereq_errors:
for error in prereq_errors:
rows.append((f"{env_mode} prerequisite", "fail", error))
diff --git a/src/skillevaluator/tier3/evals_config.py b/src/skillevaluator/tier3/evals_config.py
index 36170913..dc1219da 100644
--- a/src/skillevaluator/tier3/evals_config.py
+++ b/src/skillevaluator/tier3/evals_config.py
@@ -9,7 +9,11 @@
from __future__ import annotations
+import json
+import math
import re
+import sys
+from itertools import pairwise
from pathlib import Path
from typing import Any
@@ -50,11 +54,31 @@
"agents",
}
HARBOR_TASK_SOURCES = {"auto", "evals_json", "native_harbor"}
+# Harbor 0.22 multiplies its 600-second default verifier/build timeout by
+# this value. Keep that mandatory lifecycle timeout finite at every public
+# SkillEvaluator boundary.
+MAX_HARBOR_TIMEOUT_MULTIPLIER = sys.float_info.max / 600.0
_AGENT_KEYS = {"model"}
_RESOURCE_KEYS = {"cpus", "memory_mb", "storage_mb"}
_SKILL_WORKSPACE_KEYS = {"mode", "include"}
_GRADING_KEYS = {"mode"}
_ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_URI_AUTHORITY_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.-]{0,31}://(?P
[^\s/?#]*)")
+_SCHEMELESS_CREDENTIAL_AUTHORITY_RE = re.compile(r"[^\s/:@]+:[^\s/@]+@[^\s/?#]+")
+_REFERENCE_KWARG_NAMES_BY_ENV_MODE = {
+ "ack": frozenset({"image_pull_secret"}),
+ "cwsandbox": frozenset({"secrets"}),
+ "daytona": frozenset({"secrets"}),
+ "modal": frozenset({"registry_secret", "secrets"}),
+ "skypilot": frozenset({"secrets"}),
+ "wandb": frozenset({"secrets"}),
+}
+_CWSANDBOX_SECRET_REFERENCE_KEYS = frozenset({"env_var", "field", "name", "store"})
+_KUBERNETES_DNS_LABEL_RE = re.compile(r"^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$")
+_MAX_ENVIRONMENT_KWARGS_JSON_BYTES = 64 * 1024
+_MAX_ENVIRONMENT_KWARGS_DEPTH = 32
+_MAX_ENVIRONMENT_KWARGS_NODES = 4096
+_MAX_ENVIRONMENT_REFERENCE_BYTES = 512
class EvalsConfigError(ValueError):
@@ -152,7 +176,11 @@ def _validate_config(raw: dict[str, Any], config_path: Path) -> dict[str, Any]:
harbor["max_agents"] = _int_at_least(harbor_raw["max_agents"], 1, config_path, "harbor.max_agents")
if "timeout_multiplier" in harbor_raw:
harbor["timeout_multiplier"] = _float_greater_than(
- harbor_raw["timeout_multiplier"], 0.0, config_path, "harbor.timeout_multiplier"
+ harbor_raw["timeout_multiplier"],
+ 0.0,
+ config_path,
+ "harbor.timeout_multiplier",
+ maximum=MAX_HARBOR_TIMEOUT_MULTIPLIER,
)
if "agent_runtime_preflight" in harbor_raw:
harbor["agent_runtime_preflight"] = _bool(
@@ -241,18 +269,33 @@ def _int_at_least(value: Any, minimum: int, config_path: Path, field: str) -> in
def _float_between(value: Any, minimum: float, maximum: float, config_path: Path, field: str) -> float:
if isinstance(value, bool) or not isinstance(value, int | float):
raise EvalsConfigError(f"{config_path}: {field} must be a number")
- value = float(value)
- if not minimum <= value <= maximum:
+ try:
+ value = float(value)
+ except OverflowError:
+ raise EvalsConfigError(f"{config_path}: {field} must be between {minimum} and {maximum}") from None
+ if not math.isfinite(value) or not minimum <= value <= maximum:
raise EvalsConfigError(f"{config_path}: {field} must be between {minimum} and {maximum}")
return value
-def _float_greater_than(value: Any, minimum: float, config_path: Path, field: str) -> float:
+def _float_greater_than(
+ value: Any,
+ minimum: float,
+ config_path: Path,
+ field: str,
+ *,
+ maximum: float | None = None,
+) -> float:
if isinstance(value, bool) or not isinstance(value, int | float):
raise EvalsConfigError(f"{config_path}: {field} must be a number")
- value = float(value)
- if value <= minimum:
- raise EvalsConfigError(f"{config_path}: {field} must be > {minimum}")
+ try:
+ value = float(value)
+ except OverflowError:
+ raise EvalsConfigError(f"{config_path}: {field} must be a finite number > {minimum}") from None
+ if maximum is not None and value > maximum:
+ raise EvalsConfigError(f"{config_path}: {field} must yield finite Harbor timeouts")
+ if not math.isfinite(value) or value <= minimum:
+ raise EvalsConfigError(f"{config_path}: {field} must be a finite number > {minimum}")
return value
@@ -300,6 +343,255 @@ def _non_empty_string(value: Any, config_path: Path, field: str) -> str:
return value.strip()
+def _is_sensitive_environment_kwarg_name(name: str) -> bool:
+ """Recognize secret-bearing names across snake, kebab, and camel case."""
+ normalized = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", name)
+ normalized = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", normalized)
+ tokens = tuple(part for part in re.sub(r"[^A-Za-z0-9]+", "_", normalized).lower().split("_") if part)
+ if any(
+ token
+ in {
+ "auth",
+ "authorization",
+ "authentication",
+ "credential",
+ "credentials",
+ "passwd",
+ "password",
+ "secret",
+ "secrets",
+ "token",
+ }
+ for token in tokens
+ ):
+ return True
+ if any(
+ pair in {("api", "key"), ("access", "key"), ("private", "key"), ("secret", "key")} for pair in pairwise(tokens)
+ ):
+ return True
+ compact = "".join(tokens)
+ return compact.endswith(
+ (
+ "apikey",
+ "accesskey",
+ "privatekey",
+ "secretkey",
+ "auth",
+ "authorization",
+ "authentication",
+ "credential",
+ "credentials",
+ "passwd",
+ "password",
+ "secret",
+ "secrets",
+ "token",
+ )
+ )
+
+
+def _contains_credential_bearing_uri(value: str) -> bool:
+ """Reject URI userinfo without parsing or rendering the supplied value."""
+ if any("@" in match.group("authority") for match in _URI_AUTHORITY_RE.finditer(value)):
+ return True
+ return _SCHEMELESS_CREDENTIAL_AUTHORITY_RE.search(value) is not None
+
+
+def _safe_environment_kwarg_path(path: tuple[str | int, ...]) -> str:
+ """Describe a value location without echoing attacker-controlled nested keys."""
+ if not path:
+ return "mapping"
+ root = path[0]
+ if not isinstance(root, str) or not _ENV_NAME_RE.fullmatch(root):
+ return "mapping value"
+ return root if len(path) == 1 else f"{root} nested value"
+
+
+def _environment_kwarg_shape_error(value: Any) -> str | None:
+ """Validate bounded JSON shape iteratively so cycles/deep values fail closed."""
+ stack: list[tuple[bool, Any, tuple[str | int, ...], int]] = [(True, value, (), 0)]
+ active_containers: set[int] = set()
+ node_count = 0
+ while stack:
+ entering, current, path, depth = stack.pop()
+ if not entering:
+ active_containers.remove(id(current))
+ continue
+ node_count += 1
+ if node_count > _MAX_ENVIRONMENT_KWARGS_NODES:
+ return f"must contain at most {_MAX_ENVIRONMENT_KWARGS_NODES} JSON values"
+ if depth > _MAX_ENVIRONMENT_KWARGS_DEPTH:
+ return f"must nest at most {_MAX_ENVIRONMENT_KWARGS_DEPTH} levels"
+ if isinstance(current, dict | list):
+ identity = id(current)
+ if identity in active_containers:
+ return "must not contain cyclic values"
+ active_containers.add(identity)
+ stack.append((False, current, path, depth))
+ if isinstance(current, dict):
+ items = list(current.items())
+ for raw_key, item in reversed(items):
+ if not isinstance(raw_key, str):
+ return f"{_safe_environment_kwarg_path(path)} keys must be strings"
+ stack.append((True, item, (*path, raw_key), depth + 1))
+ else:
+ for index in range(len(current) - 1, -1, -1):
+ stack.append((True, current[index], (*path, index), depth + 1))
+ continue
+ if isinstance(current, str):
+ if _contains_credential_bearing_uri(current):
+ return (
+ f"{_safe_environment_kwarg_path(path)} contains a credential-bearing URI; "
+ "pass credentials through the host environment instead"
+ )
+ continue
+ if current is None or isinstance(current, bool | int):
+ continue
+ if isinstance(current, float):
+ if not math.isfinite(current):
+ return f"{_safe_environment_kwarg_path(path)} must be a finite JSON number"
+ continue
+ return f"{_safe_environment_kwarg_path(path)} must contain only JSON-compatible values"
+ return None
+
+
+def _secret_reference_name_error(value: Any, *, label: str) -> str | None:
+ if not isinstance(value, str) or not value.strip():
+ return f"{label} must be a non-empty secret reference name"
+ if value != value.strip() or any(ord(character) < 32 or ord(character) == 127 for character in value):
+ return f"{label} must be a trimmed secret reference name without control characters"
+ if len(value.encode("utf-8")) > _MAX_ENVIRONMENT_REFERENCE_BYTES:
+ return f"{label} must encode to at most {_MAX_ENVIRONMENT_REFERENCE_BYTES} bytes"
+ if _contains_credential_bearing_uri(value):
+ return f"{label} must be a secret reference name, not a credential-bearing URI"
+ return None
+
+
+def _reference_kwarg_error(env_mode: str | None, name: str, value: Any) -> str | None:
+ """Validate Harbor fields whose values name provider-managed secrets."""
+ if name == "image_pull_secret":
+ if env_mode != "ack":
+ return "image_pull_secret is supported only for Harbor environment 'ack'"
+ if not isinstance(value, str) or len(value.encode("utf-8")) > 253:
+ return "image_pull_secret must be a Kubernetes DNS-subdomain Secret name"
+ labels = value.split(".")
+ if not labels or any(len(label) > 63 or not _KUBERNETES_DNS_LABEL_RE.fullmatch(label) for label in labels):
+ return "image_pull_secret must be a Kubernetes DNS-subdomain Secret name"
+ return None
+ if env_mode in {"modal", "skypilot"} and name == "secrets":
+ if not isinstance(value, list):
+ return f"{name} must be a list of provider secret reference names for Harbor environment '{env_mode}'"
+ for item in value:
+ if error := _secret_reference_name_error(item, label=name):
+ return error
+ return None
+ if env_mode == "modal" and name == "registry_secret":
+ return _secret_reference_name_error(value, label=name)
+ if env_mode == "daytona" and name == "secrets":
+ if not isinstance(value, dict):
+ return "secrets must map sandbox environment variable names to Daytona organization secret names"
+ for target_name, secret_name in value.items():
+ if not isinstance(target_name, str) or not _ENV_NAME_RE.fullmatch(target_name):
+ return "secrets keys must be valid sandbox environment variable names"
+ if error := _secret_reference_name_error(secret_name, label="secrets value"):
+ return error
+ return None
+ if env_mode in {"cwsandbox", "wandb"} and name == "secrets":
+ if not isinstance(value, list):
+ return f"secrets must be a list of provider secret reference mappings for Harbor environment '{env_mode}'"
+ for item in value:
+ if not isinstance(item, dict) or not item:
+ return "secrets entries must be non-empty provider secret reference mappings"
+ if set(item) - _CWSANDBOX_SECRET_REFERENCE_KEYS:
+ return "secrets entries contain unsupported provider secret reference fields"
+ for field_name, field_value in item.items():
+ if field_name == "env_var":
+ if not isinstance(field_value, str) or not _ENV_NAME_RE.fullmatch(field_value):
+ return "secrets env_var fields must be valid environment variable names"
+ elif error := _secret_reference_name_error(field_value, label=f"secrets {field_name} field"):
+ return error
+ return None
+ return f"{name} is secret-bearing; pass credentials through the host environment instead"
+
+
+def _environment_kwarg_secret_policy_error(value: dict[str, Any], *, env_mode: str | None) -> str | None:
+ stack: list[tuple[dict[str, Any] | list[Any], tuple[str | int, ...]]] = [(value, ())]
+ allowed_mode_references = _REFERENCE_KWARG_NAMES_BY_ENV_MODE.get(env_mode or "", frozenset())
+ while stack:
+ current, path = stack.pop()
+ if isinstance(current, dict):
+ for raw_key, item in current.items():
+ item_path = (*path, raw_key)
+ is_top_level_reference = not path and (raw_key in allowed_mode_references)
+ if is_top_level_reference:
+ if error := _reference_kwarg_error(env_mode, raw_key, item):
+ return error
+ continue
+ if _is_sensitive_environment_kwarg_name(raw_key):
+ return (
+ f"{_safe_environment_kwarg_path(item_path)} is secret-bearing; "
+ "pass credentials through the host environment instead"
+ )
+ if isinstance(item, dict | list):
+ stack.append((item, item_path))
+ else:
+ for index, item in enumerate(current):
+ if isinstance(item, dict | list):
+ stack.append((item, (*path, index)))
+ return None
+
+
+def validate_environment_kwargs(value: Any, *, env_mode: str | None = None) -> dict[str, Any]:
+ """Validate non-secret Harbor constructor kwargs for safe argv forwarding."""
+ if not isinstance(value, dict):
+ raise ValueError("must be a mapping")
+ for raw_name in value:
+ if not isinstance(raw_name, str) or not _ENV_NAME_RE.fullmatch(raw_name):
+ raise ValueError("keys must be valid Python keyword names")
+ if error := _environment_kwarg_shape_error(value):
+ raise ValueError(error)
+ if error := _environment_kwarg_secret_policy_error(value, env_mode=env_mode):
+ raise ValueError(error)
+ try:
+ encoded = json.dumps(value, allow_nan=False, ensure_ascii=False, separators=(",", ":"))
+ except (RecursionError, TypeError, ValueError):
+ raise ValueError("must contain only JSON-compatible values") from None
+ if len(encoded.encode("utf-8")) > _MAX_ENVIRONMENT_KWARGS_JSON_BYTES:
+ raise ValueError(f"must encode to at most {_MAX_ENVIRONMENT_KWARGS_JSON_BYTES} bytes")
+ return dict(value)
+
+
+def parse_environment_kwarg_overrides(
+ values: tuple[str, ...] | list[str],
+ *,
+ env_mode: str | None = None,
+) -> dict[str, Any]:
+ """Parse repeatable CLI ``KEY=VALUE`` arguments using Harbor's value rules."""
+ parsed: dict[str, Any] = {}
+ for index, raw in enumerate(values, start=1):
+ if "=" not in raw:
+ raise ValueError(f"Invalid --environment-kwarg entry {index}: expected KEY=VALUE")
+ name, raw_value = raw.split("=", 1)
+ name = name.strip()
+ raw_value = raw_value.strip()
+ try:
+ value: Any = json.loads(raw_value)
+ except json.JSONDecodeError:
+ value = {"True": True, "False": False, "None": None}.get(raw_value, raw_value)
+ except RecursionError:
+ raise ValueError(f"Invalid --environment-kwarg entry {index}: JSON value nests too deeply") from None
+ parsed[name] = value
+ try:
+ return validate_environment_kwargs(parsed, env_mode=env_mode)
+ except ValueError as exc:
+ raise ValueError(f"Invalid --environment-kwarg: {exc}") from None
+
+
+def encode_environment_kwarg(name: str, value: Any) -> str:
+ """Encode one validated kwarg so Harbor's parser preserves its JSON type."""
+ return f"{name}={json.dumps(value, allow_nan=False, ensure_ascii=False, separators=(',', ':'))}"
+
+
def _resources(value: Any, config_path: Path) -> dict[str, int]:
if not isinstance(value, dict):
raise EvalsConfigError(f"{config_path}: harbor.resources must be a mapping")
diff --git a/src/skillevaluator/tier3/evals_spec.py b/src/skillevaluator/tier3/evals_spec.py
index 20b89268..890860dd 100644
--- a/src/skillevaluator/tier3/evals_spec.py
+++ b/src/skillevaluator/tier3/evals_spec.py
@@ -105,6 +105,7 @@ class EntrySpec:
"passthrough_env and setup_commands are accepted as compatibility aliases for runtime_env and pre_agent_setup.",
"Supported skill_workspace keys: mode (isolated|group), include.",
"Supported grading keys: mode (default|default_plus_custom|custom_only).",
+ "pre_agent_setup/setup_commands is allowed only with --skip-baseline; paired runs reject skill-owned setup code.",
"CLI flags override config values.",
],
example="""\
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index 1d15bef4..52fbd837 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -10,7 +10,7 @@
Each dataset entry becomes one Harbor task directory with:
instruction.md, task.toml, environment/Dockerfile or environment/skills,
- tests/eval.py, tests/entry.json
+ tests/skill_evaluator/eval.py, tests/entry.json
"""
from __future__ import annotations
@@ -34,7 +34,12 @@
from typing import Any
from urllib.parse import unquote
-from skillevaluator.tier3.case_ids import safe_child, validate_case_ids, validate_output_directory_path
+from skillevaluator.tier3.case_ids import (
+ safe_child,
+ validate_case_id,
+ validate_case_ids,
+ validate_output_directory_path,
+)
from skillevaluator.tier3.harbor.secure_copy import (
copy_file_secure,
copytree_secure,
@@ -50,12 +55,14 @@
)
from skillevaluator.tier3.toml_utils import toml_quote
from skillevaluator.utils.process_environment import child_process_env
+from skillevaluator.utils.redaction import is_sensitive_key, redact_sensitive_text
from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
logger = logging.getLogger(__name__)
TEMPLATES_DIR = Path(__file__).parent / "templates"
_EVAL_CORE_DIR = Path(__file__).resolve().parent.parent / "eval_core"
+_EVALUATOR_TESTS_SUBDIR = "skill_evaluator"
_BASE_IMAGE_PREFIX = "skillevaluator-base"
_MAX_REPO_CONTEXT_FILE_BYTES = 10 * 1024 * 1024
_MAX_REPO_CONTEXT_TOTAL_BYTES = 200 * 1024 * 1024
@@ -202,6 +209,8 @@
_COMPOSE_ALLOWED_NETWORK_KEYS = frozenset({"attachable", "enable_ipv4", "enable_ipv6", "internal", "labels"})
_COMPOSE_ALLOWED_VOLUME_KEYS = frozenset({"labels"})
_VERIFIER_JUDGE_MODEL_ENV_VARS = frozenset({"LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"})
+_VERIFIER_JUDGE_FALLBACK_ENV_VARS = frozenset({"LLM_JUDGE_FALLBACK_MODELS"})
+_VERIFIER_JUDGE_CONTROL_ENV_VARS = _VERIFIER_JUDGE_MODEL_ENV_VARS | _VERIFIER_JUDGE_FALLBACK_ENV_VARS
_VERIFIER_PROVIDER_ENV_VARS = frozenset(
{
"SKILL_EVAL_LLM_PROVIDER",
@@ -213,30 +222,151 @@
"ANTHROPIC_API_KEY",
"ANTHROPIC_BASE_URL",
"NVIDIA_API_KEY",
+ "AWS_ACCOUNT_ID",
+ "AWS_ACCOUNT_ID_ENDPOINT_MODE",
"AWS_REGION",
"AWS_ACCESS_KEY_ID",
+ "AWS_AUTH_SCHEME_PREFERENCE",
"AWS_BEARER_TOKEN_BEDROCK",
+ "AWS_CA_BUNDLE",
"AWS_CONFIG_FILE",
"AWS_CONTAINER_AUTHORIZATION_TOKEN",
"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_CONTAINER_CREDENTIALS_FULL_URI",
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
+ "AWS_CREDENTIAL_EXPIRATION",
+ "AWS_CREDENTIAL_FILE",
+ "AWS_CSM_CLIENT_ID",
+ "AWS_CSM_ENABLED",
+ "AWS_CSM_HOST",
+ "AWS_CSM_PORT",
+ "AWS_DATA_PATH",
+ "AWS_DEFAULT_PROFILE",
"AWS_DEFAULT_REGION",
+ "AWS_DEFAULTS_MODE",
+ "AWS_DISABLE_HOST_PREFIX_INJECTION",
+ "AWS_DISABLE_REQUEST_COMPRESSION",
+ "AWS_EC2_METADATA_DISABLED",
+ "AWS_EC2_METADATA_SERVICE_ENDPOINT",
+ "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
+ "AWS_EC2_METADATA_V1_DISABLED",
+ "AWS_ENDPOINT_DISCOVERY_ENABLED",
+ "AWS_ENDPOINT_URL",
+ "AWS_ENDPOINT_URL_BEDROCK",
+ "AWS_ENDPOINT_URL_BEDROCK_RUNTIME",
+ "AWS_ENDPOINT_URL_SIGNIN",
+ "AWS_ENDPOINT_URL_SSO",
+ "AWS_ENDPOINT_URL_SSO_OIDC",
+ "AWS_ENDPOINT_URL_STS",
+ "AWS_EXECUTION_ENV",
+ "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS",
+ "AWS_IMDS_USE_IPV6",
+ "AWS_LOGIN_CACHE_DIRECTORY",
+ "AWS_MAX_ATTEMPTS",
+ "AWS_METADATA_SERVICE_NUM_ATTEMPTS",
+ "AWS_METADATA_SERVICE_TIMEOUT",
+ "AWS_NEW_RETRIES_2026",
"AWS_PROFILE",
+ "AWS_REQUEST_CHECKSUM_CALCULATION",
+ "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES",
+ "AWS_RESPONSE_CHECKSUM_VALIDATION",
+ "AWS_RETRY_MODE",
"AWS_ROLE_ARN",
"AWS_ROLE_SESSION_NAME",
"AWS_SDK_LOAD_CONFIG",
+ "AWS_SDK_UA_APP_ID",
"AWS_SECRET_ACCESS_KEY",
+ "AWS_SECURITY_TOKEN",
"AWS_SESSION_TOKEN",
"AWS_SHARED_CREDENTIALS_FILE",
+ "AWS_SIGV4A_SIGNING_REGION_SET",
+ "AWS_STS_REGIONAL_ENDPOINTS",
+ "AWS_USE_DUALSTACK_ENDPOINT",
+ "AWS_USE_FIPS_ENDPOINT",
"AWS_WEB_IDENTITY_TOKEN_FILE",
+ "BOTOCORE_TCP_KEEPALIVE",
+ }
+)
+_RUNTIME_PROCESS_CONTROL_ENV_NAMES = frozenset(
+ {
+ "ALL_PROXY",
+ "BASHOPTS",
+ "BASH_ENV",
+ "CDPATH",
+ "CLAUDE_CODE_DISABLE_POLICY_SKILLS",
+ "CLAUDE_CONFIG_DIR",
+ "CLASSPATH",
+ "COMSPEC",
+ "CODEX_HOME",
+ "CURL_CA_BUNDLE",
+ "ENV",
+ "GCONV_PATH",
+ "GEMINI_CLI_HOME",
+ "HOME",
+ "HOSTALIASES",
+ "HTTPS_PROXY",
+ "HTTP_PROXY",
+ "IFS",
+ "JAVA_TOOL_OPTIONS",
+ "LOCPATH",
+ "LUA_CPATH",
+ "LUA_INIT",
+ "LUA_PATH",
+ "NLSPATH",
+ "NO_PROXY",
+ "OPENCODE_CONFIG_DIR",
+ "PATHEXT",
+ "PATH",
+ "PERL5LIB",
+ "PERL5OPT",
+ "REQUESTS_CA_BUNDLE",
+ "RES_OPTIONS",
+ "RUBYOPT",
+ "RUBYLIB",
+ "SHELLOPTS",
+ "SSLKEYLOGFILE",
+ "SSL_CERT_DIR",
+ "SSL_CERT_FILE",
+ "SSH_AUTH_SOCK",
+ "SYSTEMROOT",
+ "TEMP",
+ "TMP",
+ "TMPDIR",
+ "USERPROFILE",
+ "WINDIR",
+ "XDG_CONFIG_HOME",
+ "XDG_RUNTIME_DIR",
+ "ZDOTDIR",
+ "_JAVA_OPTIONS",
+ "all_proxy",
+ "http_proxy",
+ "https_proxy",
+ "no_proxy",
}
)
+_RUNTIME_PROCESS_CONTROL_ENV_PREFIXES = (
+ "AWS_",
+ "BASH_FUNC_",
+ "COMPOSE_",
+ "DOCKER_",
+ "DYLD_",
+ "GIT_",
+ "HARBOR_",
+ "LD_",
+ "NODE_",
+ "OTEL_",
+ "PIP_",
+ "PYTHON",
+ "SKILL_EVAL_",
+ "SKILLEVALUATOR_",
+ "UV_",
+)
def _verifier_env_vars(runtime_env: dict[str, str] | None = None) -> tuple[str, ...]:
- """Return public provider variables explicitly staged for the verifier."""
- return tuple(sorted(set(runtime_env or {}).intersection(_VERIFIER_PROVIDER_ENV_VARS)))
+ """Return evaluator-owned variables explicitly staged for the verifier."""
+ staged_controls = _VERIFIER_PROVIDER_ENV_VARS | _VERIFIER_JUDGE_FALLBACK_ENV_VARS
+ return tuple(sorted(set(runtime_env or {}).intersection(staged_controls)))
def _verifier_env_block(runtime_env: dict[str, str] | None = None, indent: str = "") -> str:
@@ -1883,6 +2013,16 @@ def _pre_agent_setup_command(pre_agent_setup: list[str] | None) -> str:
return "bash -lc " + shlex.quote(script)
+def _reject_baseline_pre_agent_setup(*, with_skill: bool, pre_agent_setup: list[str] | None) -> None:
+ """Keep skill-owned setup code out of the observational baseline arm."""
+ if with_skill or not _pre_agent_setup_command(pre_agent_setup):
+ return
+ raise ValueError(
+ "harbor.pre_agent_setup/setup_commands cannot be used while staging a baseline; "
+ "use with_skill=True (the --skip-baseline path)"
+ )
+
+
def _pre_agent_setup_healthcheck_toml_block(pre_agent_setup: list[str] | None) -> str:
command = _pre_agent_setup_command(pre_agent_setup)
if not command:
@@ -1929,34 +2069,63 @@ def _write_entry_json(
def _write_test_sh(task_dir: Path, *, grading_mode: str, custom_grader: bool) -> None:
tests_dir = task_dir / "tests"
tests_dir.mkdir(parents=True, exist_ok=True)
- prefix = '#!/bin/bash\nset -euo pipefail\ntests_dir="${HARBOR_TESTS_DIR:-/tests}"\n'
+ prefix = (
+ '#!/bin/bash\nset -euo pipefail\ntests_dir="${HARBOR_TESTS_DIR:-/tests}"\n'
+ f'evaluator_dir="${{tests_dir}}/{_EVALUATOR_TESTS_SUBDIR}"\n'
+ )
if grading_mode == "custom_only":
- script = prefix + 'python3 "${tests_dir}/custom_grader_runner.py" --mode custom_only\n'
+ script = prefix + 'python3 -I "${evaluator_dir}/custom_grader_runner.py" --mode custom_only\n'
elif grading_mode == "default_plus_custom" and custom_grader:
script = (
prefix
- + 'python3 "${tests_dir}/eval.py"\n'
- + 'python3 "${tests_dir}/custom_grader_runner.py" --mode default_plus_custom\n'
+ + 'python3 -I "${evaluator_dir}/eval.py"\n'
+ + 'python3 -I "${evaluator_dir}/custom_grader_runner.py" --mode default_plus_custom\n'
)
else:
- script = prefix + 'python3 "${tests_dir}/eval.py"\n'
+ script = prefix + 'python3 -I "${evaluator_dir}/eval.py"\n'
test_sh = tests_dir / "test.sh"
test_sh.write_text(script, encoding="utf-8")
test_sh.chmod(0o755)
+def _evaluator_tests_dir(task_dir: Path) -> Path:
+ return task_dir / "tests" / _EVALUATOR_TESTS_SUBDIR
+
+
+def _replace_evaluator_tests_dir(task_dir: Path) -> Path:
+ """Replace the evaluator-owned verifier payload inside a staged task."""
+ evaluator_dir = _evaluator_tests_dir(task_dir)
+ if os.path.lexists(evaluator_dir):
+ try:
+ metadata = evaluator_dir.lstat()
+ except OSError as exc:
+ raise ValueError(f"Cannot inspect evaluator verifier payload path: {evaluator_dir}") from exc
+ if _path_is_link_or_reparse(evaluator_dir, metadata) or not stat.S_ISDIR(metadata.st_mode):
+ evaluator_dir.unlink()
+ else:
+ if not getattr(shutil.rmtree, "avoids_symlink_attacks", False):
+ raise ValueError(
+ "cannot safely replace the evaluator verifier payload: this platform does not provide "
+ "symlink-attack-resistant recursive deletion"
+ )
+ shutil.rmtree(evaluator_dir)
+ evaluator_dir.mkdir(parents=True)
+ return evaluator_dir
+
+
def _copy_verifier(task_dir: Path) -> None:
- """Copy the standalone eval.py verifier into the task's tests/ directory."""
+ """Copy verifier code into a clean evaluator-owned tests subdirectory."""
tests_dir = task_dir / "tests"
tests_dir.mkdir(parents=True, exist_ok=True)
+ evaluator_dir = _replace_evaluator_tests_dir(task_dir)
src = TEMPLATES_DIR / "eval.py"
if src.exists():
- shutil.copy2(src, tests_dir / "eval.py")
+ shutil.copy2(src, evaluator_dir / "eval.py")
else:
logger.warning("Verifier template not found at %s", src)
lc = _EVAL_CORE_DIR / "log_converters.py"
if lc.exists():
- shutil.copy2(lc, tests_dir / "log_converters.py")
+ shutil.copy2(lc, evaluator_dir / "log_converters.py")
else:
logger.warning("log_converters helper not found at %s", lc)
@@ -1983,7 +2152,9 @@ def _copy_custom_grader(
runner_src = TEMPLATES_DIR / "custom_grader_runner.py"
if runner_src.exists():
- shutil.copy2(runner_src, tests_dir / "custom_grader_runner.py")
+ evaluator_dir = _evaluator_tests_dir(task_dir)
+ evaluator_dir.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(runner_src, evaluator_dir / "custom_grader_runner.py")
evals_dir = evals_dir or skill_path / "evals"
grader_candidates = [
@@ -2959,6 +3130,7 @@ def _validate_and_sanitize_custom_compose(
compose_path: Path,
*,
allowed_env: set[str],
+ sanitize: bool = True,
) -> None:
"""Reject Docker-host escape features and remove sidecar host ports.
@@ -3022,7 +3194,7 @@ def _validate_and_sanitize_custom_compose(
field=f"service '{svc_name}' environment",
)
- if svc_name != "main" and "ports" in service:
+ if sanitize and svc_name != "main" and "ports" in service:
del service["ports"]
changed = True
logger.debug("Stripped host port mapping from sidecar service '%s'", svc_name)
@@ -3061,7 +3233,7 @@ def _validate_and_sanitize_custom_compose(
_validate_compose_interpolation(content, allowed_env)
- if changed:
+ if changed and sanitize:
compose_path.write_text(yaml.dump(content, default_flow_style=False), encoding="utf-8")
@@ -4218,9 +4390,8 @@ def _native_entry_id(task_dir: Path) -> str:
except Exception:
return task_dir.name
metadata = data.get("metadata", {}) if isinstance(data, dict) else {}
- if isinstance(metadata, dict) and metadata.get("entry_id"):
- return str(metadata["entry_id"])
- return task_dir.name
+ raw_entry_id = metadata.get("entry_id", task_dir.name) if isinstance(metadata, dict) else task_dir.name
+ return validate_case_id(raw_entry_id)
def _environment_reference_names(value: object) -> set[str]:
@@ -4260,7 +4431,383 @@ def _validate_native_agent_judge_model_controls(task_toml: Path, environment_env
)
-def _native_task_workdir(task_dir: Path, *, allow_docker_image: bool = False) -> str | None:
+_HARBOR_HOST_ENV_TEMPLATE_RE = re.compile(r"\$\{(?P[^}:]+)(?::-(?P.*))?\}", re.DOTALL)
+
+
+def _validate_native_host_env_mapping(
+ task_toml: Path,
+ field: str,
+ environment: object,
+ *,
+ allowed_references: set[str],
+) -> None:
+ """Reject task-authored parent-env reads outside one staged role."""
+ if environment is None:
+ return
+ if not isinstance(environment, dict):
+ raise ValueError(f"Native Harbor task [{field}] must be a table: {task_toml}")
+ for key, value in environment.items():
+ if not isinstance(key, str) or not isinstance(value, str):
+ raise ValueError(f"Native Harbor task [{field}] must contain string assignments: {task_toml}")
+ template = _HARBOR_HOST_ENV_TEMPLATE_RE.fullmatch(value)
+ if template is not None:
+ source_name = template.group("name")
+ if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", source_name) is None or source_name not in allowed_references:
+ raise ValueError(
+ f"Native Harbor task [{field}] contains a host environment template "
+ f"outside the evaluator-staged role boundary: {task_toml}"
+ )
+ # Harbor substitutes a missing-source default verbatim; it does not
+ # recursively expand or otherwise validate that authored fallback.
+ default = template.group("default")
+ if default is not None and (
+ is_sensitive_key(key)
+ or redact_sensitive_text(default) != default
+ or _environment_reference_names(default)
+ ):
+ raise ValueError(
+ f"Native Harbor task [{field}] contains a literal credential-bearing assignment: {task_toml}"
+ )
+ continue
+ if is_sensitive_key(key) or redact_sensitive_text(value) != value:
+ raise ValueError(
+ f"Native Harbor task [{field}] contains a literal credential-bearing assignment: {task_toml}"
+ )
+
+
+def _validate_native_task_host_env_boundaries(
+ task_toml: Path,
+ data: dict[str, Any],
+ *,
+ runtime_env: dict[str, str] | None,
+ verifier_env: dict[str, str] | None,
+) -> None:
+ """Validate every Harbor 0.22 task field that resolves from parent env."""
+ runtime_names = set(runtime_env or {})
+ verifier_names = set(verifier_env if verifier_env is not None else runtime_env or {})
+ verifier_names.update(_VERIFIER_JUDGE_CONTROL_ENV_VARS)
+
+ environment = data.get("environment", {})
+ if not isinstance(environment, dict):
+ raise ValueError(f"Native Harbor task [environment] must be a table: {task_toml}")
+ _validate_native_host_env_mapping(
+ task_toml,
+ "environment.env",
+ environment.get("env", {}),
+ allowed_references=runtime_names,
+ )
+ solution = data.get("solution", {})
+ if not isinstance(solution, dict):
+ raise ValueError(f"Native Harbor task [solution] must be a table: {task_toml}")
+ _validate_native_host_env_mapping(
+ task_toml,
+ "solution.env",
+ solution.get("env", {}),
+ allowed_references=runtime_names,
+ )
+
+ def validate_verifier(verifier: object, field: str) -> None:
+ if not isinstance(verifier, dict):
+ raise ValueError(f"Native Harbor task [{field}] must be a table: {task_toml}")
+ _validate_native_host_env_mapping(
+ task_toml,
+ f"{field}.env",
+ verifier.get("env", {}),
+ allowed_references=verifier_names,
+ )
+ verifier_environment = verifier.get("environment")
+ if verifier_environment is None:
+ return
+ if not isinstance(verifier_environment, dict):
+ raise ValueError(f"Native Harbor task [{field}.environment] must be a table: {task_toml}")
+ _validate_native_host_env_mapping(
+ task_toml,
+ f"{field}.environment.env",
+ verifier_environment.get("env", {}),
+ allowed_references=verifier_names,
+ )
+
+ validate_verifier(data.get("verifier", {}), "verifier")
+ steps = data.get("steps")
+ if steps is None:
+ return
+ if not isinstance(steps, list):
+ raise ValueError(f"Native Harbor task [[steps]] must be an array of tables: {task_toml}")
+ for index, step in enumerate(steps):
+ if not isinstance(step, dict):
+ raise ValueError(f"Native Harbor task [[steps]] must contain tables: {task_toml}")
+ validate_verifier(step.get("verifier", {}), f"steps[{index}].verifier")
+
+
+def _validate_native_verifier_compose_boundaries(
+ task_dir: Path,
+ data: dict[str, Any],
+ *,
+ runtime_env: dict[str, str] | None,
+ verifier_env: dict[str, str] | None,
+ sanitize: bool,
+) -> None:
+ """Keep separate-verifier Compose models inside the verifier env boundary."""
+ from harbor.models.task.config import TaskConfig
+ from harbor.models.task.paths import TaskPaths
+ from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config
+
+ allowed_env = set(verifier_env if verifier_env is not None else runtime_env or {})
+ allowed_env.update(_VERIFIER_JUDGE_CONTROL_ENV_VARS)
+ config = TaskConfig.model_validate(data)
+ paths = TaskPaths(task_dir)
+ contexts: list[Path] = []
+ if config.steps:
+ for step in config.steps:
+ if resolve_effective_verifier_env_config(config, step) is None:
+ continue
+ step_context = paths.step_tests_dir(step.name)
+ if not _path_is_canonically_contained(step_context, paths.steps_dir):
+ raise ValueError(f"Native Harbor task step verifier path escapes the task: {task_dir / 'task.toml'}")
+ contexts.append(step_context if step_context.exists() else paths.tests_dir)
+ elif resolve_effective_verifier_env_config(config, step_cfg=None) is not None:
+ contexts.append(paths.tests_dir)
+
+ for context in dict.fromkeys(contexts):
+ if _has_symlink_component(context, paths.task_dir):
+ raise ValueError(f"Native Harbor verifier environment cannot use linked paths: {context}")
+ compose_path = _custom_compose_path(context)
+ if compose_path is not None:
+ if _path_is_link_or_reparse(compose_path) or not _path_is_canonically_contained(compose_path, context):
+ raise ValueError(f"Native Harbor verifier Compose file must be contained and regular: {compose_path}")
+ _validate_and_sanitize_custom_compose(
+ compose_path,
+ allowed_env=allowed_env,
+ sanitize=sanitize,
+ )
+
+
+def _native_path_has_content(path: Path) -> bool:
+ """Return whether an authored native-task path can project any payload."""
+ try:
+ metadata = path.lstat()
+ except FileNotFoundError:
+ return False
+ except OSError as exc:
+ raise ValueError(f"Cannot inspect native Harbor task path: {path}") from exc
+ if _path_is_link_or_reparse(path, metadata) or not stat.S_ISDIR(metadata.st_mode):
+ return True
+ try:
+ return next(path.iterdir(), None) is not None
+ except OSError as exc:
+ raise ValueError(f"Cannot inspect native Harbor task directory: {path}") from exc
+
+
+def _validated_native_task_model(task_dir: Path, data: dict[str, Any]) -> Any:
+ from harbor.models.task.config import TaskConfig
+
+ task_toml = task_dir / "task.toml"
+ try:
+ return TaskConfig.model_validate(data)
+ except Exception as exc:
+ raise ValueError(f"Cannot validate native Harbor task config: {task_toml}") from exc
+
+
+def _validated_native_task_steps(task_dir: Path, config: Any) -> tuple[Any, list[Any]]:
+ from harbor.models.task.paths import TaskPaths
+
+ paths = TaskPaths(task_dir)
+ steps = list(config.steps or [])
+ for step in steps:
+ if not _path_is_canonically_contained(paths.step_dir(step.name), paths.steps_dir):
+ raise ValueError(f"Native Harbor task step path escapes the task: {task_dir / 'task.toml'}")
+ return paths, steps
+
+
+def _effective_native_verifiers(config: Any, steps: list[Any]) -> list[tuple[str, Any | None]]:
+ from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config
+
+ scopes = [("task", None)] if not steps else [(f"step {step.name!r}", step) for step in steps]
+ return [(scope, resolve_effective_verifier_env_config(config, step)) for scope, step in scopes]
+
+
+def _native_custom_only_tests_are_complete(task_dir: Path) -> bool:
+ """Return whether Harbor can resolve an authored test script for every pass."""
+ config = _validated_native_task_model(
+ task_dir,
+ tomllib.loads((task_dir / "task.toml").read_text(encoding="utf-8")),
+ )
+ paths, steps = _validated_native_task_steps(task_dir, config)
+ if not steps:
+ return paths.discovered_test_path_for(config.environment.os) is not None
+
+ from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config
+
+ for step in steps:
+ verifier_environment = resolve_effective_verifier_env_config(config, step)
+ task_os = verifier_environment.os if verifier_environment is not None else config.environment.os
+ step_test = paths.discovered_step_test_path_for(step.name, task_os)
+ if verifier_environment is not None and paths.step_tests_dir(step.name).exists():
+ if step_test is None:
+ return False
+ continue
+ if step_test is None and paths.discovered_test_path_for(task_os) is None:
+ return False
+ return True
+
+
+def _reject_native_windows_execution(
+ task_toml: Path,
+ config: Any,
+ effective_verifiers: list[tuple[str, Any | None]],
+) -> None:
+ from harbor.models.task.config import TaskOS
+
+ if config.environment.os == TaskOS.WINDOWS:
+ raise ValueError(
+ "Native Harbor agent environments using Windows are unsupported until "
+ f"SkillEvaluator's task projection and verifier scripts are OS-aware: {task_toml}"
+ )
+ for scope, verifier_environment in effective_verifiers:
+ if verifier_environment is not None and verifier_environment.os == TaskOS.WINDOWS:
+ raise ValueError(
+ f"Native Harbor effective verifier for {scope} uses Windows, which is unsupported until "
+ f"SkillEvaluator's verifier projection is OS-aware: {task_toml}"
+ )
+
+
+def _reject_native_baseline_setup(task_toml: Path, config: Any, paths: Any, steps: list[Any]) -> None:
+ prior_trajectory = paths.step_trajectory_path(steps[0].name) if steps else paths.trajectory_path
+ if os.path.lexists(prior_trajectory):
+ raise ValueError(
+ f"Native Harbor task-shipped prior trajectory is not allowed in a paired baseline: {prior_trajectory}"
+ )
+ if config.environment.healthcheck is not None:
+ raise ValueError(f"Native Harbor environment healthcheck is not allowed in a paired baseline: {task_toml}")
+ for step in steps:
+ if step.healthcheck is not None:
+ raise ValueError(f"Native Harbor step healthcheck is not allowed in a paired baseline: {task_toml}")
+ step_workdir = paths.step_dir(step.name) / "workdir"
+ if os.path.lexists(step_workdir):
+ raise ValueError(
+ f"Native Harbor step workdir projection is not allowed in a paired baseline: {step_workdir}"
+ )
+
+
+def _native_standard_grader_environment_controls(
+ environment: dict[str, str],
+ *,
+ allowed_operator_references: set[str] | None = None,
+) -> set[str]:
+ """Return task-owned names that can redirect or poison the standard verifier."""
+ operator_controls = _VERIFIER_PROVIDER_ENV_VARS | _VERIFIER_JUDGE_CONTROL_ENV_VARS
+ reserved = (
+ operator_controls
+ | _RUNTIME_PROCESS_CONTROL_ENV_NAMES
+ | _RUNTIME_DISCOVERY_ENV_NAMES
+ | _RUNTIME_LOADER_ENV_NAMES
+ )
+ controls: set[str] = set()
+ for name, value in environment.items():
+ upper_name = name.upper()
+ if (
+ upper_name in operator_controls
+ and name in (allowed_operator_references or set())
+ and value == f"${{{name}}}"
+ ):
+ continue
+ if (
+ upper_name in reserved
+ or upper_name.startswith("HARBOR_")
+ or upper_name.startswith(_RUNTIME_LOADER_ENV_PREFIXES)
+ or upper_name.startswith(_RUNTIME_PROCESS_CONTROL_ENV_PREFIXES)
+ ):
+ controls.add(name)
+ return controls
+
+
+def _reject_native_standard_grading_overrides(
+ task_toml: Path,
+ grading_mode: str,
+ config: Any,
+ paths: Any,
+ steps: list[Any],
+ effective_verifiers: list[tuple[str, Any | None]],
+ allowed_verifier_env: set[str],
+) -> None:
+ if grading_mode not in {"default", "default_plus_custom"}:
+ return
+ verifier_environments = [
+ ("environment.env", config.environment.env, set()),
+ ("verifier.env", config.verifier.env, allowed_verifier_env),
+ *((f"step {step.name!r} verifier.env", step.verifier.env, allowed_verifier_env) for step in steps),
+ ]
+ for scope, environment, allowed_operator_references in verifier_environments:
+ controls = sorted(
+ _native_standard_grader_environment_controls(
+ environment,
+ allowed_operator_references=allowed_operator_references,
+ )
+ )
+ if controls:
+ raise ValueError(
+ "Native Harbor standard grading rejects task-controlled environment control(s), "
+ "including judge controls, "
+ f"in {scope}: {', '.join(controls)}: {task_toml}"
+ )
+ if config.verifier.collect:
+ raise ValueError(f"Native Harbor verifier collect hook is incompatible with standard grading: {task_toml}")
+ for step in steps:
+ if step.verifier.collect:
+ raise ValueError(
+ f"Native Harbor step verifier collect hook is incompatible with standard grading: {task_toml}"
+ )
+ step_tests = paths.step_tests_dir(step.name)
+ if _native_path_has_content(step_tests):
+ raise ValueError(
+ "Native Harbor step tests overlay is incompatible with SkillEvaluator standard grading; "
+ f"keep {step_tests} empty or use grading.mode=custom_only"
+ )
+ for scope, verifier_environment in effective_verifiers:
+ if verifier_environment is not None:
+ raise ValueError(
+ "Native Harbor separate verifier context is unsupported with SkillEvaluator standard grading; "
+ f"use a shared verifier for {scope} or grading.mode=custom_only: {task_toml}"
+ )
+
+
+def _validate_native_task_execution_compatibility(
+ task_dir: Path,
+ data: dict[str, Any],
+ *,
+ with_skill: bool,
+ grading_mode: str,
+ allowed_verifier_env: set[str],
+) -> None:
+ """Fail closed where Harbor 0.22 execution can bypass evaluator projections."""
+ task_toml = task_dir / "task.toml"
+ config = _validated_native_task_model(task_dir, data)
+ paths, steps = _validated_native_task_steps(task_dir, config)
+ effective_verifiers = _effective_native_verifiers(config, steps)
+ _reject_native_windows_execution(task_toml, config, effective_verifiers)
+ if not with_skill:
+ _reject_native_baseline_setup(task_toml, config, paths, steps)
+ _reject_native_standard_grading_overrides(
+ task_toml,
+ grading_mode,
+ config,
+ paths,
+ steps,
+ effective_verifiers,
+ allowed_verifier_env,
+ )
+
+
+def _native_task_workdir(
+ task_dir: Path,
+ *,
+ with_skill: bool,
+ grading_mode: str,
+ allow_docker_image: bool = False,
+ runtime_env: dict[str, str] | None = None,
+ verifier_env: dict[str, str] | None = None,
+ sanitize_verifier_compose: bool = False,
+) -> str | None:
"""Read and validate the workdir Harbor will use for a native task."""
try:
import tomllib
@@ -4271,10 +4818,30 @@ def _native_task_workdir(task_dir: Path, *, allow_docker_image: bool = False) ->
environment = data.get("environment", {}) if isinstance(data, dict) else {}
if not isinstance(environment, dict):
raise ValueError(f"Native Harbor task [environment] must be a table: {task_dir / 'task.toml'}")
+ _validate_native_task_execution_compatibility(
+ task_dir,
+ data,
+ with_skill=with_skill,
+ grading_mode=grading_mode,
+ allowed_verifier_env=set(verifier_env if verifier_env is not None else runtime_env or {}),
+ )
environment_env = environment.get("env", {})
if not isinstance(environment_env, dict):
raise ValueError(f"Native Harbor task [environment.env] must be a table: {task_dir / 'task.toml'}")
_validate_native_agent_judge_model_controls(task_dir / "task.toml", environment_env)
+ _validate_native_task_host_env_boundaries(
+ task_dir / "task.toml",
+ data,
+ runtime_env=runtime_env,
+ verifier_env=verifier_env,
+ )
+ _validate_native_verifier_compose_boundaries(
+ task_dir,
+ data,
+ runtime_env=runtime_env,
+ verifier_env=verifier_env,
+ sanitize=sanitize_verifier_compose,
+ )
_validate_runtime_discovery_env(environment_env)
_validate_runtime_loader_env(environment_env)
skills_dir = environment.get("skills_dir")
@@ -4296,117 +4863,72 @@ def _native_task_workdir(task_dir: Path, *, allow_docker_image: bool = False) ->
return _validated_agent_workdir(workdir)
+def _mutate_native_task_toml(
+ task_toml: Path,
+ mutation: Callable[[dict[str, Any]], None],
+) -> None:
+ """Mutate a staged native task structurally and revalidate with Harbor."""
+ try:
+ import toml
+ from harbor.models.task.config import TaskConfig
+
+ content = task_toml.read_text(encoding="utf-8")
+ TaskConfig.model_validate_toml(content)
+ data = tomllib.loads(content)
+ mutation(data)
+ rendered = toml.dumps(data)
+ TaskConfig.model_validate_toml(rendered)
+ except Exception as exc:
+ raise ValueError(f"Cannot safely update native Harbor task config: {task_toml}") from exc
+ task_toml.write_text(rendered, encoding="utf-8")
+
+
def _ensure_native_skills_dir(task_dir: Path) -> None:
"""Pin native tasks to the evaluator-owned runtime skill projection."""
- task_toml = task_dir / "task.toml"
- content = task_toml.read_text(encoding="utf-8")
- lines = content.splitlines()
- if "[environment]" in lines:
- environment_index = lines.index("[environment]")
- section_end = next(
- (index for index in range(environment_index + 1, len(lines)) if lines[index].strip().startswith("[")),
- len(lines),
- )
- if any(line.strip().startswith("skills_dir") for line in lines[environment_index + 1 : section_end]):
- return
- lines.insert(environment_index + 1, 'skills_dir = "/workspace/skills"')
- else:
- lines.extend(["", "[environment]", 'skills_dir = "/workspace/skills"'])
- task_toml.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+ def update(data: dict[str, Any]) -> None:
+ environment = data.setdefault("environment", {})
+ if not isinstance(environment, dict):
+ raise TypeError("environment is not a table")
+ environment["skills_dir"] = "/workspace/skills"
+
+ _mutate_native_task_toml(task_dir / "task.toml", update)
def _ensure_skill_evaluator_verifier_env(task_dir: Path, *, verifier_env: dict[str, str] | None) -> None:
"""Ensure staged native tasks forward configured public provider variables."""
- task_toml = task_dir / "task.toml"
- content = task_toml.read_text(encoding="utf-8")
- env_lines = [f'{name} = "${{{name}}}"' for name in _verifier_env_vars(verifier_env)]
- if all(line in content for line in env_lines):
+ env_names = _verifier_env_vars(verifier_env)
+ if not env_names:
return
- lines = content.splitlines()
- if "[verifier.env]" in lines:
- idx = lines.index("[verifier.env]") + 1
- existing = set(lines)
- for line in reversed(env_lines):
- if line not in existing:
- lines.insert(idx, line)
- task_toml.write_text("\n".join(lines) + "\n", encoding="utf-8")
- return
+ def update(data: dict[str, Any]) -> None:
+ verifier_missing = "verifier" not in data
+ verifier = data.setdefault("verifier", {})
+ if not isinstance(verifier, dict):
+ raise TypeError("verifier is not a table")
+ if verifier_missing:
+ verifier["timeout_sec"] = 180.0
+ environment = verifier.setdefault("env", {})
+ if not isinstance(environment, dict):
+ raise TypeError("verifier.env is not a table")
+ environment.update({name: f"${{{name}}}" for name in env_names})
- env_block = ["[verifier.env]", *env_lines]
- if "[verifier]" in lines:
- start = lines.index("[verifier]") + 1
- insert_at = len(lines)
- for idx in range(start, len(lines)):
- line = lines[idx].strip()
- if line.startswith("[") and line.endswith("]"):
- insert_at = idx
- break
- lines[insert_at:insert_at] = ["", *env_block]
- task_toml.write_text("\n".join(lines) + "\n", encoding="utf-8")
- return
-
- insert_at = lines.index("[environment]") if "[environment]" in lines else len(lines)
- lines[insert_at:insert_at] = ["[verifier]", "timeout_sec = 180.0", "", *env_block, ""]
- task_toml.write_text("\n".join(lines) + "\n", encoding="utf-8")
-
-
-def _insert_table_block(lines: list[str], anchor: str, block: list[str]) -> None:
- """Insert a TOML table block after *anchor* and before the next table."""
- if anchor not in lines:
- lines.extend(["", anchor])
- start = lines.index(anchor) + 1
- insert_at = len(lines)
- for idx in range(start, len(lines)):
- stripped = lines[idx].strip()
- if stripped.startswith("[") and stripped.endswith("]"):
- insert_at = idx
- break
- prefix = [] if insert_at == 0 or (insert_at > 0 and lines[insert_at - 1] == "") else [""]
- lines[insert_at:insert_at] = [*prefix, *block]
+ _mutate_native_task_toml(task_dir / "task.toml", update)
def _ensure_environment_env(task_dir: Path, runtime_env: dict[str, str]) -> None:
runtime_env = {**runtime_env, **_EVALUATOR_MANAGED_RUNTIME_ENV}
- task_toml = task_dir / "task.toml"
- lines = task_toml.read_text(encoding="utf-8").splitlines()
- header = "[environment.env]"
- rendered = {key: f"{_toml_quote(key)} = {_toml_quote(runtime_env[key])}" for key in sorted(runtime_env)}
- env_lines = list(rendered.values())
- if header in lines:
- idx = lines.index(header) + 1
- env_end = len(lines)
- for end_idx in range(idx, len(lines)):
- stripped = lines[end_idx].strip()
- if stripped.startswith("[") and stripped.endswith("]"):
- env_end = end_idx
- break
- seen_keys: set[str] = set()
- updated_section: list[str] = []
- for line in lines[idx:env_end]:
- assignment = line.split("=", 1)[0].strip() if line.strip() and "=" in line else ""
- matching_key = next(
- (key for key in runtime_env if assignment in {key, _toml_quote(key)}),
- None,
- )
- if matching_key is not None:
- if matching_key in seen_keys:
- continue
- updated_section.append(rendered[matching_key])
- seen_keys.add(matching_key)
- else:
- updated_section.append(line)
- for key, line in rendered.items():
- if key not in seen_keys:
- updated_section.append(line)
- lines[idx:env_end] = updated_section
- task_toml.write_text("\n".join(lines) + "\n", encoding="utf-8")
- return
+ def update(data: dict[str, Any]) -> None:
+ environment = data.setdefault("environment", {})
+ if not isinstance(environment, dict):
+ raise TypeError("environment is not a table")
+ environment_env = environment.setdefault("env", {})
+ if not isinstance(environment_env, dict):
+ raise TypeError("environment.env is not a table")
+ environment_env.update(runtime_env)
- block = [header, *env_lines]
- _insert_table_block(lines, "[environment]", block)
- task_toml.write_text("\n".join(lines) + "\n", encoding="utf-8")
+ _mutate_native_task_toml(task_dir / "task.toml", update)
def _ensure_pre_agent_setup_healthcheck(task_dir: Path, pre_agent_setup: list[str]) -> None:
@@ -4415,23 +4937,27 @@ def _ensure_pre_agent_setup_healthcheck(task_dir: Path, pre_agent_setup: list[st
return
task_toml = task_dir / "task.toml"
- lines = task_toml.read_text(encoding="utf-8").splitlines()
- header = "[environment.healthcheck]"
- if header in lines:
+
+ def update(data: dict[str, Any]) -> None:
+ environment = data.setdefault("environment", {})
+ if not isinstance(environment, dict):
+ raise TypeError("environment is not a table")
+ if environment.get("healthcheck") is not None:
+ raise ValueError("native task already defines an environment healthcheck")
+ environment["healthcheck"] = {
+ "command": command,
+ "interval_sec": 5.0,
+ "timeout_sec": 120.0,
+ "retries": 1,
+ }
+
+ try:
+ _mutate_native_task_toml(task_toml, update)
+ except ValueError as exc:
raise ValueError(
f"{task_toml}: harbor.pre_agent_setup cannot be injected because "
- "the native Harbor task already defines [environment.healthcheck]"
- )
-
- block = [
- header,
- f"command = {_toml_quote(command)}",
- "interval_sec = 5.0",
- "timeout_sec = 120.0",
- "retries = 1",
- ]
- _insert_table_block(lines, "[environment.env]" if "[environment.env]" in lines else "[environment]", block)
- task_toml.write_text("\n".join(lines) + "\n", encoding="utf-8")
+ "the native Harbor task already defines [environment.healthcheck] or is invalid"
+ ) from exc
def _ensure_runtime_env_and_pre_agent_setup(
@@ -4597,7 +5123,13 @@ def _stage_native_harbor_tasks_into(
raise ValueError(f"No Harbor task directories with task.toml found in {native_dir}")
validate_case_ids(path.name for path in source_task_dirs)
for source_task_dir in source_task_dirs:
- _native_task_workdir(source_task_dir)
+ _native_task_workdir(
+ source_task_dir,
+ with_skill=with_skill,
+ grading_mode=grading_mode,
+ runtime_env=runtime_env,
+ verifier_env=verifier_env,
+ )
_ = task_resources
_ = agent_workdir
@@ -4628,7 +5160,14 @@ def _stage_native_harbor_tasks_into(
baseline_aliases_prevalidated = True
for task_dir in task_dirs:
entry_id = _native_entry_id(task_dir)
- native_agent_workdir = _native_task_workdir(task_dir)
+ native_agent_workdir = _native_task_workdir(
+ task_dir,
+ with_skill=with_skill,
+ grading_mode=grading_mode,
+ runtime_env=runtime_env,
+ verifier_env=verifier_env,
+ sanitize_verifier_compose=True,
+ )
_ensure_native_skills_dir(task_dir)
entry = entries_by_id.get(entry_id)
if grading_mode in ("default", "default_plus_custom") and entry is None:
@@ -4642,13 +5181,21 @@ def _stage_native_harbor_tasks_into(
tests_dir = task_dir / "tests"
tests_dir.mkdir(parents=True, exist_ok=True)
- _copy_verifier(task_dir)
- shutil.copy2(TEMPLATES_DIR / "custom_grader_runner.py", tests_dir / "custom_grader_runner.py")
-
custom_grader = (tests_dir / "grader.py").exists() or (tests_dir / "grader.sh").exists()
- if (not custom_grader and grading_mode != "custom_only") or (
- not custom_grader and not (tests_dir / "test.sh").exists()
- ):
+ authored_custom_tests_complete = grading_mode == "custom_only" and _native_custom_only_tests_are_complete(
+ task_dir
+ )
+ evaluator_payload_required = (
+ grading_mode in ("default", "default_plus_custom") or custom_grader or not authored_custom_tests_complete
+ )
+ if evaluator_payload_required:
+ _copy_verifier(task_dir)
+ shutil.copy2(
+ TEMPLATES_DIR / "custom_grader_runner.py",
+ _evaluator_tests_dir(task_dir) / "custom_grader_runner.py",
+ )
+
+ if not custom_grader and (grading_mode != "custom_only" or not authored_custom_tests_complete):
custom_grader = _copy_custom_grader(task_dir, skill_path, grading_mode, evals_dir=evals_dir)
if grading_mode in ("default", "default_plus_custom"):
@@ -4678,9 +5225,15 @@ def _stage_native_harbor_tasks_into(
custom_grader=True,
)
_write_test_sh(task_dir, grading_mode=grading_mode, custom_grader=True)
- elif not (tests_dir / "test.sh").exists():
+ elif not authored_custom_tests_complete:
+ raise FileNotFoundError(
+ f"custom_only native Harbor task '{entry_id}' requires tests/grader.py or a Harbor-resolvable "
+ "test script for every verifier pass"
+ )
+ if grading_mode == "custom_only" and not _native_custom_only_tests_are_complete(task_dir):
raise FileNotFoundError(
- f"custom_only native Harbor task '{entry_id}' requires tests/grader.py or tests/test.sh"
+ f"custom_only native Harbor task '{entry_id}' requires tests/grader.py or a Harbor-resolvable "
+ "test script for every verifier pass"
)
if entry is not None and _entry_declares_task_input(entry, evals_dir / "files"):
@@ -4761,6 +5314,7 @@ def stage_native_harbor_tasks(
_baseline_alias_validation: _BaselineAliasValidation | None = None,
) -> list[Path]:
"""Stage native tasks privately, then publish one exact output snapshot."""
+ _reject_baseline_pre_agent_setup(with_skill=with_skill, pre_agent_setup=pre_agent_setup)
if evaluator_skill_path is None:
with private_evaluator_skill_snapshot(skill_path, task_source="native_harbor") as private_skill_path:
@@ -5159,8 +5713,8 @@ def _native_projection_entry_ids(native_dir: Path) -> tuple[str, ...]:
)
data = tomllib.loads(payload.decode("utf-8"))
metadata = data.get("metadata", {}) if isinstance(data, dict) else {}
- if isinstance(metadata, dict) and metadata.get("entry_id"):
- entry_id = str(metadata["entry_id"])
+ if isinstance(metadata, dict) and "entry_id" in metadata:
+ entry_id = validate_case_id(metadata["entry_id"])
except (UnicodeError, ValueError):
# Staging owns the detailed invalid-task diagnostic after the secure
# snapshot exists. Directory-name fallback remains non-invasive.
@@ -5440,6 +5994,7 @@ def generate_harbor_tasks(
_baseline_alias_validation: _BaselineAliasValidation | None = None,
) -> list[Path]:
"""Generate tasks from one private evals snapshot, then publish exactly."""
+ _reject_baseline_pre_agent_setup(with_skill=with_skill, pre_agent_setup=pre_agent_setup)
if evaluator_skill_path is None:
if find_evals_file(skill_path) is None:
diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.py
index c3aa6738..65297624 100644
--- a/src/skillevaluator/tier3/harbor/collector.py
+++ b/src/skillevaluator/tier3/harbor/collector.py
@@ -8,7 +8,9 @@
from __future__ import annotations
+import contextlib
import copy
+import hashlib
import json
import logging
import math
@@ -17,36 +19,83 @@
import shutil
import stat
import sys
+import unicodedata
+from dataclasses import dataclass
+from dataclasses import field as dataclass_field
from fractions import Fraction
-from pathlib import Path
+from pathlib import Path, PurePosixPath, PureWindowsPath
from typing import Any
+from harbor.models.trajectories import Trajectory
+
from skillevaluator.tier3.eval_core.atif_helpers import extract_tool_calls_as_dicts, get_skill_tool_calls
from skillevaluator.tier3.eval_core.checks import check_negative_case
from skillevaluator.tier3.harbor.metrics import (
+ CUSTOM_ONLY_METRIC_SET,
DEFAULT_METRIC_SET,
DEFAULT_METRICS,
LEGACY_METRIC_SET,
LEGACY_METRICS,
+ MAX_CUSTOM_METRICS,
+ RESERVED_METRIC_NAMES,
+ CustomMetricContractError,
average_custom_metrics,
average_metrics,
+ custom_metric_contract_error,
+ custom_metric_name_is_publishable,
dimension_scores,
extract_custom_metrics,
metric_set_for_reward,
metric_value,
overall_score,
+ rewards_have_mixed_metric_contracts,
score_definition,
+ score_value,
)
from skillevaluator.tier3.output_provenance import write_output_file_atomically
-from skillevaluator.utils.redaction import is_sensitive_key, redact_sensitive_data, redact_sensitive_text
+from skillevaluator.utils.redaction import contains_credential_value, redact_sensitive_data, redact_sensitive_text
from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot, stat_is_link_or_reparse
logger = logging.getLogger(__name__)
DISPLAY_METRICS = DEFAULT_METRICS
+_LOGICAL_ATTEMPT_SENTINEL = object()
+_CUSTOM_METRIC_CONTRACT_MARKER = "_skill_evaluator_custom_metric_contract_error"
+TRUNCATED_AGGREGATE_ATTEMPT_PREFIX = "__skillevaluator_attempt"
+_MAX_JSON_SAFE_INTEGER = (1 << 53) - 1
DEFAULT_DIAGNOSTIC_ARTIFACT_MAX_BYTES = 5 * 1024 * 1024
DIAGNOSTIC_ARTIFACT_HARD_MAX_BYTES = 64 * 1024 * 1024
REWARD_DIAGNOSTIC_STRING_MAX_CHARS = 8192
+REWARD_METADATA_TEXT_MAX_CHARS = 512
+REWARD_IDENTITY_TEXT_MAX_BYTES = 512
+REWARD_JSON_MAX_DEPTH = 64
+REWARD_JSON_MAX_NODES = 50_000
+GENERATED_JSON_MAX_BYTES = 2 * 1024 * 1024
+# Reserve room for collector-owned identity, model, and evidence annotations so
+# a source reward that passes extraction remains readable after publication.
+COLLECTED_REWARD_JSON_MAX_NODES = REWARD_JSON_MAX_NODES - 512
+COLLECTED_REWARD_JSON_MAX_BYTES = GENERATED_JSON_MAX_BYTES - (256 * 1024)
+ATIF_JSON_MAX_DEPTH = 64
+ATIF_JSON_MAX_NODES = 50_000
+PORTABLE_TRIAL_COMPONENT_MAX_UNITS = 240
+PUBLISHED_CASE_DETAILS_MAX = 256
+PUBLISHED_ATTEMPT_DETAILS_MAX = 512
+PUBLISHED_ATTEMPT_DETAILS_PER_CASE_MAX = 8
+PUBLISHED_CASE_ID_DIAGNOSTIC_SAMPLE_MAX = 32
+PUBLISHED_FAILURE_DETAILS_MAX = 32
+PUBLISHED_EXECUTION_ERRORS_MAX = 256
+PUBLISHED_JOB_FAILURE_MAX_CHARS = 4096
+UNSCOREABLE_NUMERIC_REWARD_REASON = "Reward metrics are incomplete or non-finite; trial was not scored"
+UNSAFE_REWARD_STRUCTURE_REASON = "Reward payload exceeds safe structural limits; trial was not scored"
+UNSAFE_REWARD_IDENTITY_REASON = "Reward identity violates the bounded publication contract; trial was not scored"
+UNSAFE_CUSTOM_METRICS_REASON = "Custom metrics exceed the bounded publication contract; trial was not scored"
+MALFORMED_HARBOR_REWARD_REASON = "Harbor verifier rewards contain nonnumeric metric values; trial was not scored"
+UNSAFE_CUSTOM_METRIC_UNION_REASON = (
+ "Custom metric union exceeds the per condition publication limit; condition was not scored"
+)
+MISSING_MULTI_STEP_REWARD_REASON = (
+ "Authoritative multi-step verifier reward is missing; it was not reconstructed or scored"
+)
TRIAL_DIAGNOSTIC_ARTIFACTS = ("result.json", "config.json", "exception.txt", "trial.log")
AGENT_LOG_ARTIFACTS = (
"trajectory.json",
@@ -72,6 +121,12 @@
GENERATED_ROOT_ARTIFACTS = ("attempt_policy.json", "comparison.json")
_MAX_FAILED_JUDGE_SIDECARS = 64
_MAX_FAILED_JUDGE_STEP_PATHS_SCANNED = 256
+_MAX_TRAJECTORY_REFERENCE_FILES = 64
+_MAX_TRAJECTORY_REFERENCE_DEPTH = 16
+_MAX_TRAJECTORY_REFERENCE_TOTAL_BYTES = 32 * 1024 * 1024
+_MAX_TRAJECTORY_STEP_DIRECTORIES = 64
+_ATIF_TOKEN_ID_FIELDS = frozenset({"completion_token_ids", "prompt_token_ids"})
+_TRAJECTORY_NOT_PROVIDED = object()
def _is_aggregate_extra_token_key(key: str) -> bool:
@@ -107,7 +162,20 @@ def _is_aggregate_extra_token_key(key: str) -> bool:
"ProviderException",
}
_UNCONDITIONAL_AGENT_RUNTIME_EXCEPTION_TYPES = {
+ "AgentAuthenticationError",
"AgentTimeoutError",
+ "ApiConnectionClosedError",
+ "ApiInternalServerError",
+ "ApiOverloadedError",
+ "ApiProviderResourceNotFoundError",
+ "ApiRateLimitError",
+ "ApiResponseStalledError",
+ "ApiUsageLimitError",
+ "ContextWindowExceededError",
+ "ModelNotFoundError",
+ "NetworkConnectionError",
+ "OutputTokenExceededError",
+ "UnknownApiError",
}
@@ -169,9 +237,21 @@ def _remove_generated_output_path(path: Path, output_root: Path) -> None:
def _write_generated_root_json(path: Path, output_root: Path, payload: Any) -> None:
- """Publish one root artifact without following unsafe replacements."""
+ """Publish one bounded generated artifact without following replacements."""
_assert_safe_generated_output_path(path, output_root, follow_target=False)
- write_output_file_atomically(path, json.dumps(payload, indent=2).encode("utf-8"))
+ _validate_generated_json_value(
+ payload,
+ max_depth=REWARD_JSON_MAX_DEPTH,
+ max_nodes=REWARD_JSON_MAX_NODES,
+ max_bytes=GENERATED_JSON_MAX_BYTES,
+ )
+ encoded = json.dumps(
+ payload,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ).encode("utf-8")
+ write_output_file_atomically(path, encoded)
def _agent_generated_output_paths(agent_dir: Path) -> list[Path]:
@@ -203,14 +283,9 @@ def _reset_agent_generated_outputs(agent_dir: Path, output_root: Path) -> None:
def _find_job_dir(jobs_dir: Path, job_name: str) -> Path | None:
- """Find a Harbor job directory by name."""
+ """Find the exact Harbor job directory produced for ``job_name``."""
candidate = jobs_dir / job_name
- if candidate.exists():
- return candidate
- for d in sorted(jobs_dir.iterdir(), reverse=True):
- if d.is_dir() and job_name in d.name:
- return d
- return None
+ return candidate if candidate.is_dir() else None
def _safe_text(value: Any, *, max_len: int | None = 2048) -> str:
@@ -225,6 +300,123 @@ def _safe_diagnostic_text(value: Any, *, max_len: int) -> str:
return re.sub(r"[\x00-\x1f\x7f]", " ", _safe_text(value, max_len=max_len)).strip()
+def _published_job_failure(value: Any) -> str:
+ """Return one validation/launch failure safe for generated results."""
+ return _safe_diagnostic_text(value, max_len=PUBLISHED_JOB_FAILURE_MAX_CHARS)
+
+
+def _identity_text_is_publishable(value: object) -> bool:
+ """Return whether an identity can safely be used as a generated JSON key."""
+ if not isinstance(value, str) or not value or value != value.strip() or not value.isprintable():
+ return False
+ try:
+ encoded = value.encode("utf-8")
+ except UnicodeError:
+ return False
+ return len(encoded) <= REWARD_IDENTITY_TEXT_MAX_BYTES and not contains_credential_value(value)
+
+
+def _published_trial_label(value: object, *, alias_ordinal: int | None = None) -> str:
+ """Return a bounded value-only label; unsafe identities never become keys or paths."""
+ if isinstance(value, str) and value and value.isprintable() and not contains_credential_value(value):
+ try:
+ if len(value.encode("utf-8")) <= REWARD_IDENTITY_TEXT_MAX_BYTES:
+ return value
+ except UnicodeError:
+ pass
+ if alias_ordinal is not None:
+ return f"redacted-or-invalid-trial-{alias_ordinal:06d}"
+ return "redacted-or-invalid-trial"
+
+
+def _validated_expected_case_ids(expected_case_ids: list[str] | None) -> list[str]:
+ """Validate caller-owned case identities before generated outputs are reset."""
+ validated: list[str] = []
+ seen: dict[str, str] = {}
+ for case_id in expected_case_ids or []:
+ if not _identity_text_is_publishable(case_id):
+ raise ValueError("Expected case identity violates the bounded publication contract")
+ collision_key = case_id.casefold()
+ if collision_key in seen:
+ raise ValueError(
+ "Expected case identities must be unique without cross-platform collisions: "
+ f"{case_id!r} conflicts with {seen[collision_key]!r}"
+ )
+ seen[collision_key] = case_id
+ validated.append(case_id)
+ return validated
+
+
+def _validated_case_id_by_task_selector(
+ case_id_by_task_selector: dict[str, str] | None,
+ expected_case_ids: list[str],
+) -> dict[str, str] | None:
+ """Copy one trusted staged-selector mapping after rejecting ambiguity."""
+ if case_id_by_task_selector is None:
+ return None
+
+ validated: dict[str, str] = {}
+ seen_selectors: dict[str, str] = {}
+ seen_case_ids: dict[str, str] = {}
+ for selector, case_id in case_id_by_task_selector.items():
+ if not _identity_text_is_publishable(selector) or not _identity_text_is_publishable(case_id):
+ raise ValueError("Task selector mapping violates the bounded publication contract")
+ selector_key = selector.casefold()
+ if selector_key in seen_selectors:
+ raise ValueError(
+ "Task selector mapping contains duplicate or cross-platform colliding selectors: "
+ f"{selector!r} conflicts with {seen_selectors[selector_key]!r}"
+ )
+ case_id_key = case_id.casefold()
+ if case_id_key in seen_case_ids:
+ raise ValueError(
+ "Task selector mapping must contain unique logical case identities without cross-platform collisions: "
+ f"{case_id!r} conflicts with {seen_case_ids[case_id_key]!r}"
+ )
+ seen_selectors[selector_key] = selector
+ seen_case_ids[case_id_key] = case_id
+ validated[selector] = case_id
+
+ if expected_case_ids and set(validated.values()) != set(expected_case_ids):
+ raise ValueError("Task selector mapping must match the expected logical case identities")
+ return validated
+
+
+def _sampled_case_id_diagnostic(label: str, case_ids: list[str]) -> str:
+ """Render bounded case-coverage diagnostics while retaining an exact count."""
+ if not case_ids:
+ return ""
+ sample = case_ids[:PUBLISHED_CASE_ID_DIAGNOSTIC_SAMPLE_MAX]
+ rendered = ", ".join(sample)
+ if len(case_ids) > len(sample):
+ return f"{label} (showing {len(sample)} of {len(case_ids)}): {rendered}"
+ return f"{label}: {rendered}"
+
+
+def _public_failure_list(failures: list[dict[str, str]] | None) -> list[dict[str, str]]:
+ """Return a bounded, redacted sample of trial-level failures."""
+ public: list[dict[str, str]] = []
+ for failure in (failures or [])[:PUBLISHED_FAILURE_DETAILS_MAX]:
+ public.append(
+ {
+ "trial": _published_trial_label(failure.get("trial", "unknown trial")),
+ "reason": _safe_diagnostic_text(failure.get("reason", "unknown error"), max_len=2048),
+ }
+ )
+ return public
+
+
+def _failure_list_metadata(prefix: str, failures: list[dict[str, str]] | None) -> dict[str, Any]:
+ """Describe exact failure cardinality beside a bounded published sample."""
+ total = len(failures or [])
+ shown = min(total, PUBLISHED_FAILURE_DETAILS_MAX)
+ return {
+ f"{prefix}_total": total,
+ f"{prefix}_shown": shown,
+ f"{prefix}_truncated": shown < total,
+ }
+
+
def _safe_evaluation_errors(value: Any) -> dict[str, str] | list[str] | str:
"""Normalize verifier diagnostics before persisting or displaying them."""
if isinstance(value, dict):
@@ -244,6 +436,13 @@ def _safe_evaluation_errors(value: Any) -> dict[str, str] | list[str] | str:
return _safe_diagnostic_text(value, max_len=512)
+def _bounded_reward_metadata_text(value: Any) -> str | None:
+ """Return one bounded, redacted collector-owned metadata label."""
+ if not isinstance(value, str) or not value:
+ return None
+ return _safe_diagnostic_text(value, max_len=REWARD_METADATA_TEXT_MAX_CHARS) or None
+
+
def _read_json(path: Path) -> Any:
"""Read one bounded regular JSON file through an anchored no-follow root."""
try:
@@ -502,12 +701,8 @@ def _trajectory_agent_runtime_failure_reason(trajectory: Any) -> str:
return ""
-def _trial_exception_details(trial_dir: Path) -> tuple[str, str]:
- """Return the Harbor trial exception type and display reason, if present."""
- result = _read_json(trial_dir / "result.json")
- if not isinstance(result, dict):
- return "", ""
- exception_info = result.get("exception_info")
+def _exception_details(exception_info: Any) -> tuple[str, str]:
+ """Return one Harbor exception type and bounded display reason."""
if not isinstance(exception_info, dict):
return "", ""
@@ -518,6 +713,30 @@ def _trial_exception_details(trial_dir: Path) -> tuple[str, str]:
return exception_type, (exception_type or exception_message)[:600]
+def _trial_exception_details(trial_dir: Path) -> tuple[str, str]:
+ """Return the Harbor trial-root exception type and display reason, if present."""
+ result = _read_json(trial_dir / "result.json")
+ if not isinstance(result, dict):
+ return "", ""
+ return _exception_details(result.get("exception_info"))
+
+
+def _trial_step_exception_details(trial_dir: Path) -> list[tuple[str, str]]:
+ """Return ordered native multi-step exception types and display reasons."""
+ result = _read_json(trial_dir / "result.json")
+ if not isinstance(result, dict):
+ return []
+ step_results = result.get("step_results")
+ if not isinstance(step_results, list):
+ return []
+ return [
+ details
+ for step in step_results
+ if isinstance(step, dict)
+ if (details := _exception_details(step.get("exception_info"))) != ("", "")
+ ]
+
+
def _agent_log_runtime_failure_reason(
trial_dir: Path,
*,
@@ -558,24 +777,28 @@ def _agent_log_runtime_failure_reason(
def _agent_runtime_failure_reason(trial_dir: Path) -> str:
"""Return why a trial cannot produce a valid score."""
- exception_type, exception_reason = _trial_exception_details(trial_dir)
+ exception_details = [
+ _trial_exception_details(trial_dir),
+ *_trial_step_exception_details(trial_dir),
+ ]
agent_reason = _agent_log_runtime_failure_reason(
trial_dir,
- include_text_logs=bool(exception_reason),
+ include_text_logs=any(reason for _exception_type, reason in exception_details),
)
if agent_reason:
return agent_reason
- if exception_type in _UNCONDITIONAL_AGENT_RUNTIME_EXCEPTION_TYPES:
- return exception_reason
+ for exception_type, exception_reason in exception_details:
+ if exception_type in _UNCONDITIONAL_AGENT_RUNTIME_EXCEPTION_TYPES:
+ return exception_reason
- # Do not classify verifier/healthcheck/task exceptions as agent runtime failures.
- if (
- exception_type in _AGENT_RUNTIME_EXCEPTION_TYPES
- and exception_reason
- and _text_contains_agent_runtime_failure(exception_reason)
- ):
- return exception_reason
+ # Do not classify verifier/healthcheck/task exceptions as agent runtime failures.
+ if (
+ exception_type in _AGENT_RUNTIME_EXCEPTION_TYPES
+ and exception_reason
+ and _text_contains_agent_runtime_failure(exception_reason)
+ ):
+ return exception_reason
return ""
@@ -793,16 +1016,17 @@ def _trial_failure_reason(trial_dir: Path) -> str:
def _extract_trial_failures(job_dir: Path) -> list[dict[str, str]]:
failures: list[dict[str, str]] = []
- for trial_dir in sorted(job_dir.iterdir()):
+ for ordinal, trial_dir in enumerate(sorted(job_dir.iterdir()), start=1):
+ trial_label = _published_trial_label(trial_dir.name, alias_ordinal=ordinal)
kind, unsafe_reason = _inspect_trial_directory(trial_dir)
if kind == "link":
- failures.append({"trial": trial_dir.name, "reason": unsafe_reason})
+ failures.append({"trial": trial_label, "reason": unsafe_reason})
continue
if kind != "directory":
continue
reason = _trial_failure_reason(trial_dir)
if reason:
- failures.append({"trial": trial_dir.name, "reason": redact_sensitive_text(reason)})
+ failures.append({"trial": trial_label, "reason": redact_sensitive_text(reason)})
return failures
@@ -838,13 +1062,18 @@ def _can_preserve_partial_rewards(job_dir: Path, trial_failures: list[dict[str,
def _extract_agent_runtime_failures(job_dir: Path) -> list[dict[str, str]]:
failures: list[dict[str, str]] = []
- for trial_dir in sorted(job_dir.iterdir()):
+ for ordinal, trial_dir in enumerate(sorted(job_dir.iterdir()), start=1):
kind, _reason = _inspect_trial_directory(trial_dir)
if kind != "directory":
continue
reason = _agent_runtime_failure_reason(trial_dir)
if reason:
- failures.append({"trial": trial_dir.name, "reason": redact_sensitive_text(reason)})
+ failures.append(
+ {
+ "trial": _published_trial_label(trial_dir.name, alias_ordinal=ordinal),
+ "reason": redact_sensitive_text(reason),
+ }
+ )
return failures
@@ -860,14 +1089,41 @@ def _diagnostic_artifact_max_bytes() -> int:
return min(max(0, value), DIAGNOSTIC_ARTIFACT_HARD_MAX_BYTES)
+def _without_atif_token_id_fields(value: Any) -> Any:
+ if isinstance(value, dict):
+ return {
+ str(key): _without_atif_token_id_fields(item)
+ for key, item in value.items()
+ if str(key) not in _ATIF_TOKEN_ID_FIELDS
+ }
+ if isinstance(value, list):
+ return [_without_atif_token_id_fields(item) for item in value]
+ return value
+
+
+def _redacted_trajectory_data(value: Any) -> dict[str, Any] | None:
+ """Redact one ATIF document without changing schema field types."""
+ try:
+ validated = _validated_trajectory_dict(value)
+ without_token_ids = _without_atif_token_id_fields(validated)
+ return _validated_trajectory_dict(redact_sensitive_data(without_token_ids))
+ except (_TrajectoryMergeError, RecursionError, MemoryError):
+ return None
+
+
def _redacted_artifact_text(src: Path, text: str) -> str | None:
if src.suffix.lower() == ".json":
try:
data = json.loads(text)
- safe_data = redact_sensitive_data(data)
+ if src.name.startswith("trajectory"):
+ safe_data = _redacted_trajectory_data(data)
+ if safe_data is None:
+ return None
+ else:
+ safe_data = redact_sensitive_data(data)
# Compact encoding avoids indentation-driven amplification for
# deeply nested but otherwise valid diagnostic JSON.
- return json.dumps(safe_data, separators=(",", ":"), ensure_ascii=False)
+ return json.dumps(safe_data, separators=(",", ":"), ensure_ascii=True, allow_nan=False)
except (json.JSONDecodeError, ValueError, RecursionError, MemoryError):
return None
try:
@@ -958,7 +1214,12 @@ def _write_redacted_text_copy(
return True, {"name": src.name, "size_bytes": size_bytes}
-def _copy_trial_artifacts(trial_dir: Path, trial_out: Path) -> list[str]:
+def _copy_trial_artifacts(
+ trial_dir: Path,
+ trial_out: Path,
+ *,
+ include_root_trajectory: bool = True,
+) -> list[str]:
copied: list[str] = []
manifest: dict[str, Any] = {"copied": [], "skipped": []}
for artifact_name in TRIAL_DIAGNOSTIC_ARTIFACTS:
@@ -985,6 +1246,8 @@ def _copy_trial_artifacts(trial_dir: Path, trial_out: Path) -> list[str]:
manifest["skipped"].append({"name": "agent", "reason": "not_regular_directory"})
elif agent_logs_metadata is not None:
for artifact_name in AGENT_LOG_ARTIFACTS:
+ if artifact_name == "trajectory.json" and not include_root_trajectory:
+ continue
src = agent_logs / artifact_name
if src.exists() or src.is_symlink():
ok, record = _write_redacted_text_copy(src, trial_out / artifact_name, source_root=trial_dir)
@@ -998,6 +1261,24 @@ def _copy_trial_artifacts(trial_dir: Path, trial_out: Path) -> list[str]:
return copied
+def _record_skipped_trajectory(trial_out: Path, reason: str) -> None:
+ """Persist a value-free explanation when canonical trajectory output is omitted."""
+ manifest_path = trial_out / "artifact_manifest.json"
+ existing = _read_json(manifest_path)
+ manifest = existing if isinstance(existing, dict) else {"copied": [], "skipped": []}
+ copied = manifest.get("copied")
+ skipped = manifest.get("skipped")
+ if not isinstance(copied, list):
+ manifest["copied"] = []
+ if not isinstance(skipped, list):
+ skipped = []
+ manifest["skipped"] = skipped
+ record = {"name": "trajectory.json", "reason": reason}
+ if record not in skipped:
+ skipped.append(record)
+ _write_artifact_manifest(trial_out, manifest)
+
+
def _trial_error_summary(trial_dir: Path) -> dict[str, Any]:
result_file = trial_dir / "result.json"
summary: dict[str, Any] = {}
@@ -1038,6 +1319,51 @@ def _trial_error_summary(trial_dir: Path) -> dict[str, Any]:
return summary
+def _write_bounded_failure_artifact(path: Path, failure: dict[str, Any]) -> None:
+ """Write a redacted failure record inside the generated JSON envelope."""
+ try:
+ safe_failure = redact_sensitive_data(failure, max_str_len=REWARD_METADATA_TEXT_MAX_CHARS)
+ _validate_generated_json_value(
+ safe_failure,
+ max_depth=REWARD_JSON_MAX_DEPTH,
+ max_nodes=REWARD_JSON_MAX_NODES,
+ max_bytes=GENERATED_JSON_MAX_BYTES,
+ )
+ except (MemoryError, RecursionError, TypeError, UnicodeError, ValueError):
+ raw_error = failure.get("error")
+ error = raw_error if isinstance(raw_error, dict) else {}
+ safe_failure = {
+ "status": "unscored",
+ "trial": _published_trial_label(failure.get("trial", "unknown trial")),
+ "agent": _safe_diagnostic_text(failure.get("agent", "unknown"), max_len=256),
+ "variant": _safe_diagnostic_text(failure.get("variant", "unknown"), max_len=64),
+ "artifacts": [
+ _safe_diagnostic_text(item, max_len=128)
+ for item in (failure.get("artifacts") if isinstance(failure.get("artifacts"), list) else [])[:64]
+ ],
+ "error": {
+ "type": _safe_diagnostic_text(error.get("type", "HarborTrialError"), max_len=128),
+ "message": _safe_diagnostic_text(
+ error.get("message", "Failure metadata exceeded safe artifact limits"),
+ max_len=512,
+ ),
+ },
+ "diagnostic_truncated": True,
+ "diagnostic_truncated_reason": "failure metadata exceeded safe artifact limits",
+ }
+ if str(failure.get("evaluation_status") or "").casefold() in {"error", "failed"}:
+ safe_failure["evaluation_status"] = "failed"
+ if failure.get("evaluation_errors") not in (None, ""):
+ safe_failure["evaluation_errors"] = _safe_evaluation_errors(failure["evaluation_errors"])
+ encoded = json.dumps(
+ safe_failure,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ).encode("utf-8")
+ write_output_file_atomically(path, encoded)
+
+
def _looks_like_trial_dir(path: Path) -> bool:
return any((path / name).exists() for name in TRIAL_DIAGNOSTIC_ARTIFACTS) or (path / "agent").exists()
@@ -1051,29 +1377,32 @@ def _save_unscored_trials(
variant: str,
agent_model: str | None = None,
agent_model_source: str | None = None,
+ persisted_names: dict[str, str] | None = None,
) -> None:
if job_dir is None or not job_dir.exists():
return
+ agent = _bounded_reward_metadata_text(agent) or "unknown"
+ agent_model = _bounded_reward_metadata_text(agent_model)
+ agent_model_source = _bounded_reward_metadata_text(agent_model_source)
scored_trials = {str(reward.get("_trial_root_name") or reward.get("_trial_name") or "") for reward in rewards}
+ if persisted_names is None:
+ _scored_names, persisted_names = _persisted_trial_layout(rewards, job_dir)
for trial_src in sorted(job_dir.iterdir()):
kind, unsafe_reason = _inspect_trial_directory(trial_src)
if kind == "link":
- trial_out = trials_dir / trial_src.name
+ trial_out = trials_dir / persisted_names.get(trial_src.name, "unknown")
trial_out.mkdir(parents=True, exist_ok=True)
- write_output_file_atomically(
+ _write_bounded_failure_artifact(
trial_out / "failure.json",
- json.dumps(
- {
- "status": "unscored",
- "trial": trial_src.name,
- "agent": agent,
- "variant": variant,
- "artifacts": [],
- "error": {"type": "UnsafeHarborTrial", "message": unsafe_reason},
- },
- indent=2,
- ).encode("utf-8"),
+ {
+ "status": "unscored",
+ "trial": trial_out.name,
+ "agent": agent,
+ "variant": variant,
+ "artifacts": [],
+ "error": {"type": "UnsafeHarborTrial", "message": unsafe_reason},
+ },
)
continue
if kind != "directory":
@@ -1081,9 +1410,29 @@ def _save_unscored_trials(
if trial_src.name in scored_trials or not _looks_like_trial_dir(trial_src):
continue
- trial_out = trials_dir / trial_src.name
+ trial_out = trials_dir / persisted_names.get(trial_src.name, "unknown")
trial_out.mkdir(parents=True, exist_ok=True)
- copied = _copy_trial_artifacts(trial_src, trial_out)
+ copied = _copy_trial_artifacts(trial_src, trial_out, include_root_trajectory=False)
+ materialized_trajectory, trajectory_reason = _materialized_trial_trajectory(trial_src, None)
+ safe_trajectory = (
+ _redacted_trajectory_data(materialized_trajectory) if materialized_trajectory is not None else None
+ )
+ if safe_trajectory is not None:
+ (trial_out / "trajectory.json").write_text(
+ json.dumps(
+ safe_trajectory,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ),
+ encoding="utf-8",
+ )
+ copied.append("trajectory.json")
+ else:
+ if materialized_trajectory is not None:
+ trajectory_reason = "trajectory_redaction_or_validation_failed"
+ _record_skipped_trajectory(trial_out, trajectory_reason or "trajectory_unavailable")
+ copied.append("artifact_manifest.json")
judge_diagnostic = _failed_judge_diagnostic(trial_src)
if judge_diagnostic is not None:
judge_diagnostic.update({"agent": agent, "variant": variant})
@@ -1098,7 +1447,7 @@ def _save_unscored_trials(
copied.append("reward.json")
failure = {
"status": "unscored",
- "trial": trial_src.name,
+ "trial": trial_out.name,
"agent": agent,
"variant": variant,
"artifacts": copied,
@@ -1117,11 +1466,8 @@ def _save_unscored_trials(
failure.update(error_summary)
failure_file = trial_out / "failure.json"
try:
- failure_file.write_text(
- json.dumps(redact_sensitive_data(failure), indent=2),
- encoding="utf-8",
- )
- except OSError as e:
+ _write_bounded_failure_artifact(failure_file, failure)
+ except (OSError, TypeError, ValueError) as e:
logger.debug("Failed to write Harbor failure artifact %s: %s", failure_file, e)
@@ -1166,9 +1512,13 @@ def _ordered_step_trajectory_paths(trial_root: Path) -> list[Path]:
return []
safe_paths: dict[str, Path] = {}
+ scanned_entries = 0
try:
with os.scandir(steps_dir) as entries:
for entry in entries:
+ scanned_entries += 1
+ if scanned_entries > _MAX_TRAJECTORY_STEP_DIRECTORIES:
+ return []
try:
entry_metadata = entry.stat(follow_symlinks=False)
except OSError:
@@ -1194,6 +1544,7 @@ def _ordered_step_trajectory_paths(trial_root: Path) -> list[Path]:
return []
ordered_names: list[str] = []
+ seen_names: set[str] = set()
result = _read_json(trial_root / "result.json")
if isinstance(result, dict):
step_results = result.get("step_results")
@@ -1201,8 +1552,9 @@ def _ordered_step_trajectory_paths(trial_root: Path) -> list[Path]:
for step in step_results:
if isinstance(step, dict):
step_name = step.get("step_name")
- if isinstance(step_name, str) and step_name and step_name not in ordered_names:
+ if isinstance(step_name, str) and step_name and step_name not in seen_names:
ordered_names.append(step_name)
+ seen_names.add(step_name)
ordered_paths: list[Path] = []
seen: set[Path] = set()
@@ -1218,93 +1570,1250 @@ def _ordered_step_trajectory_paths(trial_root: Path) -> list[Path]:
return ordered_paths
-def _merged_step_trajectory(trial_root: Path) -> dict[str, Any] | None:
- """Merge Harbor multi-step ATIF fragments into one collected trajectory."""
- trajectories: list[tuple[str, dict[str, Any]]] = []
- for path in _ordered_step_trajectory_paths(trial_root):
- data = _read_json(path)
- if not isinstance(data, dict):
+def _trajectory_dict_steps(trajectory: dict[str, Any]) -> list[dict[str, Any]]:
+ steps = trajectory.get("steps")
+ if not isinstance(steps, list):
+ return []
+ return [step for step in steps if isinstance(step, dict)]
+
+
+def _trajectory_step_merge_identity(
+ step: dict[str, Any],
+ *,
+ copied_context: bool = False,
+) -> dict[str, Any]:
+ """Return semantic step content without collector-owned identity fields."""
+ identity = {
+ key: value
+ for key, value in step.items()
+ if key not in {"is_copied_context", "step_id"} and not (copied_context and key == "metrics")
+ }
+ extra = identity.get("extra")
+ if isinstance(extra, dict):
+ identity["extra"] = {
+ key: value
+ for key, value in extra.items()
+ if key not in {"harbor_step_name", "harbor_original_step_id"} and not (copied_context and key == "note")
+ }
+ if not identity["extra"]:
+ identity.pop("extra")
+ return identity
+
+
+def _is_cumulative_trajectory_continuation(
+ previous: dict[str, Any],
+ current: dict[str, Any],
+) -> bool:
+ """Return true only when ATIF copied-context markers prove continuation."""
+ previous_steps = _trajectory_dict_steps(previous)
+ current_steps = _trajectory_dict_steps(current)
+ if not previous_steps or not current_steps:
+ return False
+ copied_prefix_length = _copied_context_prefix_length(current_steps)
+ if copied_prefix_length:
+ if copied_prefix_length > len(previous_steps):
+ raise _TrajectoryMergeError("copied-context prefix exceeds previous trajectory")
+ if copied_prefix_length == len(current_steps):
+ raise _TrajectoryMergeError("copied-context continuation has no new steps")
+ for previous_step, current_step in zip(
+ previous_steps[-copied_prefix_length:],
+ current_steps[:copied_prefix_length],
+ strict=True,
+ ):
+ if _trajectory_step_merge_identity(previous_step, copied_context=True) != (
+ _trajectory_step_merge_identity(current_step, copied_context=True)
+ ):
+ raise _TrajectoryMergeError("copied-context prefix does not match previous trajectory suffix")
+ return True
+
+ previous_session = previous.get("session_id")
+ current_session = current.get("session_id")
+ if (
+ not isinstance(previous_session, str)
+ or not previous_session
+ or current_session != previous_session
+ or previous_session == "copilot-cli"
+ or len(current_steps) <= len(previous_steps)
+ ):
+ return False
+ return all(
+ _trajectory_step_merge_identity(previous_step) == _trajectory_step_merge_identity(current_step)
+ for previous_step, current_step in zip(
+ previous_steps,
+ current_steps[: len(previous_steps)],
+ strict=True,
+ )
+ )
+
+
+def _copied_context_prefix_length(steps: list[dict[str, Any]]) -> int:
+ """Return the leading copied-context count, rejecting non-prefix markers."""
+ prefix_length = 0
+ saw_new_step = False
+ for step in steps:
+ if step.get("is_copied_context") is True:
+ if saw_new_step:
+ raise _TrajectoryMergeError("copied-context steps must form a prefix")
+ prefix_length += 1
+ else:
+ saw_new_step = True
+ return prefix_length
+
+
+class _TrajectoryMergeError(ValueError):
+ """A multi-file ATIF trajectory cannot be materialized without data loss."""
+
+
+@dataclass
+class _TrajectoryReferenceState:
+ raw_cache: dict[str, dict[str, Any]] = dataclass_field(default_factory=dict)
+ materialized_cache: dict[str, dict[str, Any]] = dataclass_field(default_factory=dict)
+ resolved_ids: dict[str, str] = dataclass_field(default_factory=dict)
+ reference_ordinals: dict[str, int] = dataclass_field(default_factory=dict)
+ generated_continuation_fingerprints: dict[str, str] = dataclass_field(default_factory=dict)
+ active: set[str] = dataclass_field(default_factory=set)
+ file_count: int = 0
+ total_bytes: int = 0
+
+
+def _normalized_trajectory_reference(reference: Any) -> tuple[Path, str]:
+ if not isinstance(reference, str) or not reference.strip() or len(reference) > 512 or "\x00" in reference:
+ raise _TrajectoryMergeError("invalid trajectory reference")
+ # Harbor resolves this value as an exact child path. Preserve significant
+ # leading/trailing whitespace for the source lookup; normalization is only
+ # for traversal checks and the cache key.
+ raw = reference
+ try:
+ raw.encode("utf-8")
+ except UnicodeError as exc:
+ raise _TrajectoryMergeError("invalid trajectory reference encoding") from exc
+ platform_path = PureWindowsPath(raw) if os.name == "nt" else PurePosixPath(raw)
+ if platform_path.drive or platform_path.is_absolute():
+ raise _TrajectoryMergeError("absolute trajectory reference")
+ if "://" in raw or any(part in {"", ".", ".."} for part in platform_path.parts):
+ raise _TrajectoryMergeError("unsafe trajectory reference")
+ # Match Harbor's native ``agent_dir / reference`` lookup exactly. In
+ # particular, POSIX treats a backslash as a literal filename character;
+ # rewriting it to ``/`` can select a different trajectory than Harbor did.
+ relative = Path(raw)
+ key = platform_path.as_posix().casefold() if os.name == "nt" else platform_path.as_posix()
+ return relative, key
+
+
+def _validate_generated_json_structure(
+ value: Any,
+ *,
+ max_depth: int,
+ max_nodes: int,
+) -> None:
+ """Reject trees that the bounded report loader cannot traverse."""
+ nodes = 0
+ stack: list[tuple[Any, int]] = [(value, 1)]
+ while stack:
+ current, depth = stack.pop()
+ nodes += 1
+ if nodes > max_nodes:
+ raise ValueError("JSON node count exceeds limit")
+ if not isinstance(current, dict | list):
continue
- steps = data.get("steps")
- if not isinstance(steps, list):
+ if depth > max_depth:
+ raise ValueError("JSON depth exceeds limit")
+ if nodes + len(current) > max_nodes:
+ raise ValueError("JSON node count exceeds limit")
+ children = current.values() if isinstance(current, dict) else current
+ stack.extend((child, depth + 1) for child in children)
+
+
+def _validate_generated_json_value(
+ value: Any,
+ *,
+ max_depth: int,
+ max_nodes: int,
+ max_bytes: int,
+) -> None:
+ """Validate structural, numeric, and encoded-size browser safety."""
+ _validate_generated_json_structure(value, max_depth=max_depth, max_nodes=max_nodes)
+ stack: list[Any] = [value]
+ while stack:
+ current = stack.pop()
+ if isinstance(current, dict):
+ stack.extend(current.values())
+ elif isinstance(current, list):
+ stack.extend(current)
+ elif isinstance(current, bool) or current is None or isinstance(current, str):
continue
- step_name = path.parent.parent.name
- trajectories.append((step_name, data))
+ elif isinstance(current, int):
+ if abs(current) > _MAX_JSON_SAFE_INTEGER:
+ raise ValueError("browser-unsafe JSON integer")
+ elif isinstance(current, float):
+ if not math.isfinite(current):
+ raise ValueError("non-finite JSON number")
+ else:
+ raise TypeError("value is not JSON serializable")
+ encoded = json.dumps(
+ value,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ).encode("utf-8")
+ if len(encoded) > max_bytes:
+ raise ValueError("encoded JSON exceeds limit")
+
+
+def _validated_trajectory_dict(data: Any) -> dict[str, Any]:
+ try:
+ # Bound traversal before deepcopy so hostile ATIF cannot amplify work
+ # before validation begins.
+ _validate_generated_json_structure(
+ data,
+ max_depth=ATIF_JSON_MAX_DEPTH,
+ max_nodes=ATIF_JSON_MAX_NODES,
+ )
+ candidate = copy.deepcopy(data)
+ if isinstance(candidate, dict):
+ final_metrics = candidate.get("final_metrics")
+ if isinstance(final_metrics, dict):
+ for key in (
+ "total_prompt_tokens",
+ "total_completion_tokens",
+ "total_cached_tokens",
+ "total_steps",
+ ):
+ value = final_metrics.get(key)
+ if key in final_metrics and (
+ not isinstance(value, int)
+ or isinstance(value, bool)
+ or value < 0
+ or value > _MAX_JSON_SAFE_INTEGER
+ ):
+ final_metrics.pop(key, None)
+ cost = final_metrics.get("total_cost_usd")
+ if "total_cost_usd" in final_metrics:
+ try:
+ cost_is_finite = (
+ isinstance(cost, int | float) and not isinstance(cost, bool) and math.isfinite(cost)
+ )
+ except OverflowError:
+ cost_is_finite = False
+ if not cost_is_finite:
+ final_metrics.pop("total_cost_usd", None)
+ metric_extra = final_metrics.get("extra")
+ if isinstance(metric_extra, dict):
+ for key, value in list(metric_extra.items()):
+ if _is_aggregate_extra_token_key(str(key)) and (
+ not isinstance(value, int)
+ or isinstance(value, bool)
+ or value < 0
+ or value > _MAX_JSON_SAFE_INTEGER
+ ):
+ metric_extra.pop(key, None)
+ steps = candidate.get("steps")
+ if isinstance(steps, list):
+ for step in steps:
+ if not isinstance(step, dict):
+ continue
+ llm_call_count = step.get("llm_call_count")
+ if "llm_call_count" in step and (
+ not isinstance(llm_call_count, int)
+ or isinstance(llm_call_count, bool)
+ or llm_call_count < 0
+ or llm_call_count > _MAX_JSON_SAFE_INTEGER
+ ):
+ step.pop("llm_call_count", None)
+ step_metrics = step.get("metrics")
+ if not isinstance(step_metrics, dict):
+ continue
+ for key in ("prompt_tokens", "completion_tokens", "cached_tokens"):
+ value = step_metrics.get(key)
+ if key in step_metrics and (
+ not isinstance(value, int)
+ or isinstance(value, bool)
+ or value < 0
+ or value > _MAX_JSON_SAFE_INTEGER
+ ):
+ step_metrics.pop(key, None)
+ cost = step_metrics.get("cost_usd")
+ if "cost_usd" in step_metrics:
+ try:
+ cost_is_finite = (
+ isinstance(cost, int | float) and not isinstance(cost, bool) and math.isfinite(cost)
+ )
+ except OverflowError:
+ cost_is_finite = False
+ if not cost_is_finite:
+ step_metrics.pop("cost_usd", None)
+ step_extra = step_metrics.get("extra")
+ if isinstance(step_extra, dict):
+ for key, value in list(step_extra.items()):
+ if _is_aggregate_extra_token_key(str(key)) and (
+ not isinstance(value, int)
+ or isinstance(value, bool)
+ or value < 0
+ or value > _MAX_JSON_SAFE_INTEGER
+ ):
+ step_extra.pop(key, None)
+ _validate_generated_json_value(
+ candidate,
+ max_depth=ATIF_JSON_MAX_DEPTH,
+ max_nodes=ATIF_JSON_MAX_NODES,
+ max_bytes=GENERATED_JSON_MAX_BYTES,
+ )
+ validated = Trajectory.model_validate(candidate).to_json_dict()
+ _validate_generated_json_value(
+ validated,
+ max_depth=ATIF_JSON_MAX_DEPTH,
+ max_nodes=ATIF_JSON_MAX_NODES,
+ max_bytes=GENERATED_JSON_MAX_BYTES,
+ )
+ return validated
+ except (TypeError, ValueError, RecursionError, UnicodeError, MemoryError) as exc:
+ raise _TrajectoryMergeError("invalid ATIF trajectory") from exc
+
+
+def _trajectory_reference_cache_key(agent_dir: Path, reference_key: str) -> str:
+ return f"{agent_dir.absolute()}\0{reference_key}"
+
+
+def _read_referenced_trajectory(
+ agent_dir: Path,
+ reference: Any,
+ state: _TrajectoryReferenceState,
+ *,
+ count_against_reference_budget: bool,
+) -> tuple[dict[str, Any], str]:
+ relative, key = _normalized_trajectory_reference(reference)
+ cache_key = _trajectory_reference_cache_key(agent_dir, key)
+ if cached := state.raw_cache.get(cache_key):
+ return copy.deepcopy(cached), key
+ if count_against_reference_budget and state.file_count >= _MAX_TRAJECTORY_REFERENCE_FILES:
+ raise _TrajectoryMergeError("trajectory reference count exceeded")
+ try:
+ with SecureRoot(agent_dir) as secure_root:
+ raw, _metadata = secure_root.read_bytes(relative, DEFAULT_DIAGNOSTIC_ARTIFACT_MAX_BYTES)
+ except (OSError, SecurePathError) as exc:
+ raise _TrajectoryMergeError("unreadable trajectory reference") from exc
+ if count_against_reference_budget:
+ state.file_count += 1
+ state.total_bytes += len(raw)
+ if state.total_bytes > _MAX_TRAJECTORY_REFERENCE_TOTAL_BYTES:
+ raise _TrajectoryMergeError("trajectory reference bytes exceeded")
+ try:
+ data = json.loads(raw)
+ except (UnicodeError, ValueError, RecursionError) as exc:
+ raise _TrajectoryMergeError("invalid trajectory JSON") from exc
+ validated = _validated_trajectory_dict(data)
+ state.raw_cache[cache_key] = validated
+ return copy.deepcopy(validated), key
+
- if not trajectories:
+def _agent_identity(agent: Any) -> dict[str, Any] | None:
+ if not isinstance(agent, dict):
return None
+ return {key: agent.get(key) for key in ("name", "version", "model_name", "tool_definitions")}
+
+
+def _combined_notes(*values: Any) -> str | None:
+ notes: list[str] = []
+ for value in values:
+ if isinstance(value, str) and value and value not in notes:
+ notes.append(value)
+ return "\n\n".join(notes) or None
+
- merged = copy.deepcopy(trajectories[0][1])
- merged_steps: list[dict[str, Any]] = []
- for step_name, trajectory in trajectories:
- steps = trajectory.get("steps")
- if not isinstance(steps, list):
+def _canonical_trajectory_sha256(trajectory: dict[str, Any]) -> str:
+ redacted = _redacted_trajectory_data(trajectory)
+ if redacted is None:
+ raise _TrajectoryMergeError("trajectory identity cannot be safely canonicalized")
+ canonical = json.dumps(
+ redacted,
+ sort_keys=True,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ )
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _raw_trajectory_sha256(trajectory: dict[str, Any]) -> str:
+ try:
+ canonical = json.dumps(
+ trajectory,
+ sort_keys=True,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ )
+ except (TypeError, ValueError, RecursionError, UnicodeError) as exc:
+ raise _TrajectoryMergeError("trajectory fingerprint cannot be safely canonicalized") from exc
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _is_trusted_materialized_cumulative_continuation(
+ previous: dict[str, Any],
+ current: dict[str, Any],
+ *,
+ state: _TrajectoryReferenceState,
+) -> bool:
+ """Recognize native resume across collector-built explicit continuation chains."""
+ if _is_cumulative_trajectory_continuation(previous, current):
+ return True
+ for trajectory in (previous, current):
+ trajectory_id = trajectory.get("trajectory_id")
+ if not isinstance(trajectory_id, str) or not trajectory_id:
+ return False
+ if state.generated_continuation_fingerprints.get(trajectory_id) != _raw_trajectory_sha256(trajectory):
+ return False
+ previous_steps = _trajectory_dict_steps(previous)
+ current_steps = _trajectory_dict_steps(current)
+ if not previous_steps or len(current_steps) <= len(previous_steps):
+ return False
+ return all(
+ _trajectory_step_merge_identity(previous_step) == _trajectory_step_merge_identity(current_step)
+ for previous_step, current_step in zip(
+ previous_steps,
+ current_steps[: len(previous_steps)],
+ strict=True,
+ )
+ )
+
+
+def _synthetic_trajectory_content_sha256(trajectory: dict[str, Any]) -> str:
+ """Hash redacted semantic content without source identifiers or references."""
+ redacted = _redacted_trajectory_data(trajectory)
+ if redacted is None:
+ raise _TrajectoryMergeError("trajectory identity cannot be safely canonicalized")
+ redacted.pop("continued_trajectory_ref", None)
+ redacted.pop("session_id", None)
+ redacted.pop("trajectory_id", None)
+ root_extra = redacted.get("extra")
+ if isinstance(root_extra, dict):
+ for key in ("harbor_continuation", "harbor_multi_step", "harbor_parent_scope"):
+ root_extra.pop(key, None)
+ canonical = json.dumps(
+ redacted,
+ sort_keys=True,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ )
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _merged_embedded_subagents(*collections: Any) -> list[dict[str, Any]]:
+ merged: list[dict[str, Any]] = []
+ identities: dict[str, str] = {}
+ for collection in collections:
+ if collection is None:
continue
- for step in steps:
- if not isinstance(step, dict):
+ if not isinstance(collection, list):
+ raise _TrajectoryMergeError("invalid embedded subagent collection")
+ for item in collection:
+ validated = _validated_trajectory_dict(item)
+ trajectory_id = validated.get("trajectory_id")
+ if not isinstance(trajectory_id, str) or not trajectory_id:
+ raise _TrajectoryMergeError("embedded subagent lacks trajectory_id")
+ canonical = json.dumps(validated, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+ previous = identities.get(trajectory_id)
+ if previous is not None and previous != canonical:
+ raise _TrajectoryMergeError("conflicting embedded subagent trajectory_id")
+ if previous is None:
+ identities[trajectory_id] = canonical
+ merged.append(validated)
+ return merged
+
+
+def _remap_parent_subagent_scope(
+ trajectory: dict[str, Any],
+ *,
+ namespace: str,
+) -> dict[str, Any]:
+ """Give one parent's embedded IDs a deterministic combined-parent scope."""
+ scoped = copy.deepcopy(trajectory)
+ embedded = _merged_embedded_subagents(scoped.get("subagent_trajectories"))
+ if not embedded:
+ return scoped
+
+ aliases: dict[str, str] = {}
+ remapped: list[dict[str, Any]] = []
+ for child_index, child in enumerate(embedded):
+ original_id = str(child["trajectory_id"])
+ identity = {
+ "namespace": namespace,
+ "child_index": child_index,
+ "content_sha256": _synthetic_trajectory_content_sha256(child),
+ }
+ digest = hashlib.sha256(
+ json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
+ ).hexdigest()[:20]
+ scoped_id = f"skillevaluator-scoped-subagent-{digest}"
+ aliases[original_id] = scoped_id
+ child_extra = child.get("extra")
+ if not isinstance(child_extra, dict):
+ child_extra = {}
+ prior_scope = copy.deepcopy(child_extra.get("harbor_parent_scope"))
+ scope_provenance: dict[str, Any] = {
+ "original_trajectory_id": redact_sensitive_text(original_id),
+ "parent_scope": namespace,
+ }
+ if prior_scope is not None:
+ scope_provenance["prior_scope"] = prior_scope
+ child_extra["harbor_parent_scope"] = scope_provenance
+ child["extra"] = child_extra
+ child["trajectory_id"] = scoped_id
+ remapped.append(_validated_trajectory_dict(child))
+
+ for step in _trajectory_dict_steps(scoped):
+ observation = step.get("observation")
+ if not isinstance(observation, dict):
+ continue
+ results = observation.get("results")
+ if not isinstance(results, list):
+ continue
+ for result in results:
+ if not isinstance(result, dict):
continue
- merged_step = copy.deepcopy(step)
- original_step_id = merged_step.get("step_id")
- merged_step["step_id"] = len(merged_steps) + 1
- extra = merged_step.get("extra")
- if not isinstance(extra, dict):
- extra = {}
- extra.setdefault("harbor_step_name", step_name)
- if original_step_id not in (None, ""):
- extra.setdefault("harbor_original_step_id", original_step_id)
- merged_step["extra"] = extra
- merged_steps.append(merged_step)
-
- if not merged_steps:
- return None
+ refs = result.get("subagent_trajectory_ref")
+ if not isinstance(refs, list):
+ continue
+ for ref in refs:
+ if not isinstance(ref, dict):
+ continue
+ trajectory_id = ref.get("trajectory_id")
+ if isinstance(trajectory_id, str) and trajectory_id in aliases:
+ ref["trajectory_id"] = aliases[trajectory_id]
+ scoped["subagent_trajectories"] = remapped
+ return _validated_trajectory_dict(scoped)
+
- step_names = [name for name, _ in trajectories]
- merged["steps"] = merged_steps
- merged["schema_version"] = str(trajectories[0][1].get("schema_version") or merged.get("schema_version") or "")
- merged["agent"] = trajectories[0][1].get("agent") or merged.get("agent")
- merged_extra = merged.get("extra")
- if not isinstance(merged_extra, dict):
- merged_extra = {}
- merged_extra["harbor_multi_step"] = {
- "step_count": len(step_names),
- "step_names": step_names,
+def _continuation_source_provenance(
+ trajectory: dict[str, Any],
+ *,
+ trusted_continuation_fingerprints: dict[str, str],
+) -> tuple[
+ int,
+ list[str],
+ list[str],
+ list[dict[str, Any]],
+ list[dict[str, Any]],
+ list[dict[str, Any]],
+ list[str],
+]:
+ extra = trajectory.get("extra")
+ continuation = extra.get("harbor_continuation") if isinstance(extra, dict) else None
+ trajectory_id = trajectory.get("trajectory_id")
+ trusted_fingerprint = (
+ trusted_continuation_fingerprints.get(trajectory_id) if isinstance(trajectory_id, str) else None
+ )
+ if (
+ trusted_fingerprint is not None
+ and trusted_fingerprint == _raw_trajectory_sha256(trajectory)
+ and isinstance(continuation, dict)
+ ):
+ count = continuation.get("segment_count")
+ sessions = continuation.get("source_session_ids")
+ trajectory_ids = continuation.get("source_trajectory_ids")
+ root_extras = continuation.get("source_root_extra")
+ agent_extras = continuation.get("source_agent_extra")
+ final_metrics_extras = continuation.get("source_final_metrics_extra")
+ schema_versions = continuation.get("source_schema_versions")
+ if (
+ isinstance(count, int)
+ and not isinstance(count, bool)
+ and count >= 1
+ and isinstance(sessions, list)
+ and all(isinstance(value, str) and value for value in sessions)
+ and isinstance(trajectory_ids, list)
+ and all(isinstance(value, str) and value for value in trajectory_ids)
+ and isinstance(root_extras, list)
+ and all(isinstance(value, dict) for value in root_extras)
+ and isinstance(agent_extras, list)
+ and all(isinstance(value, dict) for value in agent_extras)
+ and isinstance(final_metrics_extras, list)
+ and all(isinstance(value, dict) for value in final_metrics_extras)
+ and isinstance(schema_versions, list)
+ and all(isinstance(value, str) and value for value in schema_versions)
+ ):
+ return (
+ count,
+ list(sessions),
+ list(trajectory_ids),
+ copy.deepcopy(root_extras),
+ copy.deepcopy(agent_extras),
+ copy.deepcopy(final_metrics_extras),
+ list(schema_versions),
+ )
+ session_id = trajectory.get("session_id")
+ agent = trajectory.get("agent")
+ agent_extra = agent.get("extra") if isinstance(agent, dict) else None
+ final_metrics = trajectory.get("final_metrics")
+ final_metrics_extra = final_metrics.get("extra") if isinstance(final_metrics, dict) else None
+ schema_version = trajectory.get("schema_version")
+ return (
+ 1,
+ [session_id] if isinstance(session_id, str) and session_id else [],
+ [trajectory_id] if isinstance(trajectory_id, str) and trajectory_id else [],
+ [copy.deepcopy(extra)] if isinstance(extra, dict) else [],
+ [copy.deepcopy(agent_extra)] if isinstance(agent_extra, dict) else [],
+ [copy.deepcopy(final_metrics_extra)] if isinstance(final_metrics_extra, dict) else [],
+ [schema_version] if isinstance(schema_version, str) and schema_version else [],
+ )
+
+
+def _combine_continuation_trajectories(
+ base: dict[str, Any],
+ continuation: dict[str, Any],
+ *,
+ state: _TrajectoryReferenceState,
+) -> dict[str, Any]:
+ if _agent_identity(base.get("agent")) != _agent_identity(continuation.get("agent")):
+ raise _TrajectoryMergeError("continuation agent mismatch")
+ base_steps = _trajectory_dict_steps(base)
+ continuation_steps = _trajectory_dict_steps(continuation)
+ copied_prefix_length = _copied_context_prefix_length(continuation_steps)
+ if copied_prefix_length:
+ appended_steps = continuation_steps[copied_prefix_length:]
+ else:
+ appended_steps = continuation_steps
+
+ base_namespace = "continuation-base"
+ continuation_namespace = "continuation-next"
+ scoped_base = _remap_parent_subagent_scope(base, namespace=base_namespace)
+ scoped_continuation = _remap_parent_subagent_scope(
+ continuation,
+ namespace=continuation_namespace,
+ )
+ base_steps = _trajectory_dict_steps(scoped_base)
+ continuation_steps = _trajectory_dict_steps(scoped_continuation)
+ if copied_prefix_length:
+ appended_steps = continuation_steps[copied_prefix_length:]
+ else:
+ appended_steps = continuation_steps
+
+ combined = copy.deepcopy(scoped_base)
+ combined["schema_version"] = "ATIF-v1.7"
+ combined["agent"] = copy.deepcopy(scoped_continuation["agent"])
+ combined["steps"] = [copy.deepcopy(step) for step in (*base_steps, *appended_steps)]
+ for index, step in enumerate(combined["steps"], start=1):
+ step["step_id"] = index
+ sessions = [value for value in (base.get("session_id"), continuation.get("session_id")) if value]
+ if sessions and len(set(sessions)) == 1:
+ combined["session_id"] = sessions[0]
+ else:
+ combined.pop("session_id", None)
+ if "final_metrics" in continuation:
+ combined["final_metrics"] = copy.deepcopy(continuation["final_metrics"])
+ if isinstance(combined["final_metrics"], dict):
+ combined["final_metrics"]["total_steps"] = len(combined["steps"])
+ else:
+ combined.pop("final_metrics", None)
+ notes = _combined_notes(base.get("notes"), continuation.get("notes"))
+ if notes:
+ combined["notes"] = notes
+ else:
+ combined.pop("notes", None)
+ subagents = _merged_embedded_subagents(
+ scoped_base.get("subagent_trajectories"),
+ scoped_continuation.get("subagent_trajectories"),
+ )
+ if subagents:
+ combined["subagent_trajectories"] = subagents
+ else:
+ combined.pop("subagent_trajectories", None)
+ combined.pop("continued_trajectory_ref", None)
+ (
+ base_count,
+ base_sessions,
+ base_trajectory_ids,
+ base_root_extras,
+ base_agent_extras,
+ base_final_metrics_extras,
+ base_schema_versions,
+ ) = _continuation_source_provenance(
+ base,
+ trusted_continuation_fingerprints=state.generated_continuation_fingerprints,
+ )
+ (
+ continuation_count,
+ continuation_sessions,
+ continuation_trajectory_ids,
+ continuation_root_extras,
+ continuation_agent_extras,
+ continuation_final_metrics_extras,
+ continuation_schema_versions,
+ ) = _continuation_source_provenance(
+ continuation,
+ trusted_continuation_fingerprints=state.generated_continuation_fingerprints,
+ )
+ continuation_identity = {
+ "base_sha256": _canonical_trajectory_sha256(base),
+ "continuation_sha256": _canonical_trajectory_sha256(continuation),
}
- merged["extra"] = merged_extra
+ continuation_digest = hashlib.sha256(
+ json.dumps(continuation_identity, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ ).hexdigest()[:20]
+ combined["trajectory_id"] = f"skillevaluator-continuation-{continuation_digest}"
+ combined["extra"] = {
+ "harbor_continuation": {
+ "segment_count": base_count + continuation_count,
+ "source_session_ids": [*base_sessions, *continuation_sessions],
+ "source_trajectory_ids": [*base_trajectory_ids, *continuation_trajectory_ids],
+ "source_root_extra": [*base_root_extras, *continuation_root_extras],
+ "source_agent_extra": [*base_agent_extras, *continuation_agent_extras],
+ "source_final_metrics_extra": [
+ *base_final_metrics_extras,
+ *continuation_final_metrics_extras,
+ ],
+ "source_schema_versions": [*base_schema_versions, *continuation_schema_versions],
+ }
+ }
+ return _validated_trajectory_dict(combined)
+
+
+def _mint_embedded_trajectory_id(
+ reference_key: str,
+ trajectory: dict[str, Any],
+ *,
+ reference_ordinal: int,
+) -> str:
+ identity = {
+ "reference": redact_sensitive_text(reference_key),
+ "reference_ordinal": reference_ordinal,
+ "content_sha256": _canonical_trajectory_sha256(trajectory),
+ }
+ digest = hashlib.sha256(
+ json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
+ ).hexdigest()[:20]
+ return f"skillevaluator-subagent-{digest}"
+
- merged["final_metrics"] = _merge_trajectory_final_metrics(
- [trajectory for _, trajectory in trajectories],
- total_steps=len(merged_steps),
+def _materialized_trajectory_source_ids(
+ trajectory: dict[str, Any],
+ *,
+ state: _TrajectoryReferenceState,
+) -> set[str]:
+ values = {str(trajectory_id)} if (trajectory_id := trajectory.get("trajectory_id")) else set()
+ trusted_fingerprint = state.generated_continuation_fingerprints.get(str(trajectory_id))
+ if trusted_fingerprint != _raw_trajectory_sha256(trajectory):
+ return values
+ extra = trajectory.get("extra")
+ continuation = extra.get("harbor_continuation") if isinstance(extra, dict) else None
+ source_ids = continuation.get("source_trajectory_ids") if isinstance(continuation, dict) else None
+ if isinstance(source_ids, list):
+ values.update(str(value) for value in source_ids if isinstance(value, str) and value)
+ return values
+
+
+def _materialize_embedded_trajectory(
+ trajectory: dict[str, Any],
+ *,
+ agent_dir: Path,
+ state: _TrajectoryReferenceState,
+ depth: int,
+) -> dict[str, Any]:
+ if depth > _MAX_TRAJECTORY_REFERENCE_DEPTH:
+ raise _TrajectoryMergeError("trajectory reference depth exceeded")
+ materialized = copy.deepcopy(trajectory)
+ continued_ref = materialized.pop("continued_trajectory_ref", None)
+ combined_continuation = False
+ if continued_ref:
+ continuation, _continuation_key = _materialize_trajectory_file(
+ agent_dir,
+ continued_ref,
+ state=state,
+ depth=depth + 1,
+ )
+ materialized = _combine_continuation_trajectories(materialized, continuation, state=state)
+ combined_continuation = True
+ materialized = _resolve_subagent_trajectory_refs(
+ materialized,
+ agent_dir=agent_dir,
+ state=state,
+ depth=depth + 1,
)
- return merged
+ if combined_continuation:
+ state.generated_continuation_fingerprints[materialized["trajectory_id"]] = _raw_trajectory_sha256(materialized)
+ return materialized
+
+
+def _resolve_subagent_trajectory_refs(
+ trajectory: dict[str, Any],
+ *,
+ agent_dir: Path,
+ state: _TrajectoryReferenceState,
+ depth: int,
+) -> dict[str, Any]:
+ if depth > _MAX_TRAJECTORY_REFERENCE_DEPTH:
+ raise _TrajectoryMergeError("trajectory reference depth exceeded")
+ source_embedded = _merged_embedded_subagents(trajectory.get("subagent_trajectories"))
+ embedded: list[dict[str, Any]] = []
+ by_id: dict[str, dict[str, Any]] = {}
+ aliases: dict[str, str] = {}
+ materialized_sources: list[tuple[int, str, dict[str, Any]]] = []
+ for source_index, source in enumerate(source_embedded):
+ source_id = str(source["trajectory_id"])
+ materialized = _materialize_embedded_trajectory(
+ source,
+ agent_dir=agent_dir,
+ state=state,
+ depth=depth,
+ )
+ materialized_id = materialized.get("trajectory_id")
+ if not isinstance(materialized_id, str) or not materialized_id:
+ raise _TrajectoryMergeError("materialized embedded subagent lacks trajectory_id")
+ if state.generated_continuation_fingerprints.get(materialized_id) == _raw_trajectory_sha256(materialized):
+ identity = {
+ "embedded_ordinal": source_index,
+ "content_sha256": _canonical_trajectory_sha256(materialized),
+ }
+ digest = hashlib.sha256(
+ json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
+ ).hexdigest()[:20]
+ materialized_id = f"skillevaluator-embedded-continuation-{digest}"
+ materialized["trajectory_id"] = materialized_id
+ materialized = _validated_trajectory_dict(materialized)
+ state.generated_continuation_fingerprints[materialized_id] = _raw_trajectory_sha256(materialized)
+ materialized_sources.append((source_index, source_id, materialized))
+
+ redacted_id_counts: dict[str, int] = {}
+ for _source_index, _source_id, materialized in materialized_sources:
+ safe_id = redact_sensitive_text(str(materialized["trajectory_id"]))
+ redacted_id_counts[safe_id] = redacted_id_counts.get(safe_id, 0) + 1
+
+ for source_index, source_id, materialized in materialized_sources:
+ materialized_id = str(materialized["trajectory_id"])
+ safe_id = redact_sensitive_text(materialized_id)
+ if redacted_id_counts[safe_id] > 1:
+ was_generated_continuation = state.generated_continuation_fingerprints.get(
+ materialized_id
+ ) == _raw_trajectory_sha256(materialized)
+ identity = {
+ "embedded_ordinal": source_index,
+ "content_sha256": _synthetic_trajectory_content_sha256(materialized),
+ }
+ digest = hashlib.sha256(
+ json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
+ ).hexdigest()[:20]
+ materialized_id = f"skillevaluator-redacted-subagent-{digest}"
+ materialized["trajectory_id"] = materialized_id
+ materialized = _validated_trajectory_dict(materialized)
+ if was_generated_continuation:
+ state.generated_continuation_fingerprints[materialized_id] = _raw_trajectory_sha256(materialized)
+ aliases[source_id] = materialized_id
+ existing = by_id.get(materialized_id)
+ if existing is not None and existing != materialized:
+ raise _TrajectoryMergeError("conflicting embedded subagent trajectory_id")
+ if existing is None:
+ by_id[materialized_id] = materialized
+ embedded.append(materialized)
+
+ refs_to_resolve: list[dict[str, Any]] = []
+ for step in _trajectory_dict_steps(trajectory):
+ observation = step.get("observation")
+ if not isinstance(observation, dict):
+ continue
+ results = observation.get("results")
+ if not isinstance(results, list):
+ raise _TrajectoryMergeError("invalid trajectory observation results")
+ for result in results:
+ if not isinstance(result, dict):
+ raise _TrajectoryMergeError("invalid trajectory observation result")
+ refs = result.get("subagent_trajectory_ref")
+ if refs is None:
+ continue
+ if not isinstance(refs, list):
+ raise _TrajectoryMergeError("invalid subagent trajectory references")
+ for ref in refs:
+ if not isinstance(ref, dict):
+ raise _TrajectoryMergeError("invalid subagent trajectory reference")
+ refs_to_resolve.append(ref)
+
+ explicit_path_ids: dict[str, str] = {}
+ for ref in refs_to_resolve:
+ trajectory_path = ref.get("trajectory_path")
+ trajectory_id = ref.get("trajectory_id")
+ if not trajectory_path or not isinstance(trajectory_id, str):
+ continue
+ _relative, reference_key = _normalized_trajectory_reference(trajectory_path)
+ reference_cache_key = _trajectory_reference_cache_key(agent_dir, reference_key)
+ proposed_id = aliases.get(trajectory_id, trajectory_id)
+ prior = explicit_path_ids.get(reference_cache_key)
+ if prior is not None and prior != proposed_id:
+ raise _TrajectoryMergeError("conflicting trajectory_id aliases for referenced file")
+ explicit_path_ids[reference_cache_key] = proposed_id
+
+ for ref in refs_to_resolve:
+ trajectory_path = ref.get("trajectory_path")
+ trajectory_id = ref.get("trajectory_id")
+ embedded_id = aliases.get(trajectory_id, trajectory_id) if isinstance(trajectory_id, str) else None
+ if embedded_id in by_id:
+ # ATIF allows both keys and recommends preferring an available
+ # embedded document over an external sidecar. Remember the path
+ # alias so a later path-only reference resolves to this same ID.
+ if trajectory_path:
+ _relative, reference_key = _normalized_trajectory_reference(trajectory_path)
+ reference_cache_key = _trajectory_reference_cache_key(agent_dir, reference_key)
+ prior_resolved_id = state.resolved_ids.get(reference_cache_key)
+ if prior_resolved_id not in (None, embedded_id):
+ raise _TrajectoryMergeError("conflicting trajectory_id aliases for referenced file")
+ state.resolved_ids[reference_cache_key] = embedded_id
+ ref["trajectory_id"] = embedded_id
+ ref.pop("trajectory_path", None)
+ elif trajectory_path:
+ _relative, reference_key = _normalized_trajectory_reference(trajectory_path)
+ reference_cache_key = _trajectory_reference_cache_key(agent_dir, reference_key)
+ remembered_id = state.resolved_ids.get(reference_cache_key)
+ candidate_id = remembered_id or explicit_path_ids.get(reference_cache_key)
+ if candidate_id is not None and candidate_id in by_id:
+ proposed_id = embedded_id or explicit_path_ids.get(reference_cache_key)
+ if proposed_id not in (None, candidate_id):
+ raise _TrajectoryMergeError("conflicting trajectory_id aliases for referenced file")
+ state.resolved_ids[reference_cache_key] = candidate_id
+ ref["trajectory_id"] = candidate_id
+ ref.pop("trajectory_path", None)
+ continue
+ external, reference_key = _materialize_trajectory_file(
+ agent_dir,
+ trajectory_path,
+ state=state,
+ depth=depth + 1,
+ )
+ external_id = external.get("trajectory_id")
+ generated_external_id = bool(
+ isinstance(external_id, str)
+ and state.generated_continuation_fingerprints.get(external_id) == _raw_trajectory_sha256(external)
+ )
+ if (
+ trajectory_id
+ and external_id
+ and not generated_external_id
+ and trajectory_id not in _materialized_trajectory_source_ids(external, state=state)
+ ):
+ raise _TrajectoryMergeError("subagent trajectory_id does not match referenced file")
+ reference_cache_key = _trajectory_reference_cache_key(agent_dir, reference_key)
+ prior_resolved_id = state.resolved_ids.get(reference_cache_key)
+ proposed_id = (
+ trajectory_id
+ or explicit_path_ids.get(reference_cache_key)
+ or (None if generated_external_id else external_id)
+ )
+ if prior_resolved_id is not None and proposed_id not in (None, prior_resolved_id):
+ raise _TrajectoryMergeError("conflicting trajectory_id aliases for referenced file")
+ reference_ordinal = state.reference_ordinals.setdefault(
+ reference_cache_key,
+ len(state.reference_ordinals),
+ )
+ resolved_id = (
+ prior_resolved_id
+ or proposed_id
+ or _mint_embedded_trajectory_id(
+ reference_key,
+ external,
+ reference_ordinal=reference_ordinal,
+ )
+ )
+ state.resolved_ids[reference_cache_key] = resolved_id
+ external["trajectory_id"] = resolved_id
+ external = _validated_trajectory_dict(external)
+ if generated_external_id:
+ state.generated_continuation_fingerprints[resolved_id] = _raw_trajectory_sha256(external)
+ existing = by_id.get(resolved_id)
+ if existing is not None and existing != external:
+ raise _TrajectoryMergeError("conflicting referenced subagent trajectory_id")
+ if existing is None:
+ by_id[resolved_id] = external
+ embedded.append(external)
+ ref["trajectory_id"] = resolved_id
+ ref.pop("trajectory_path", None)
+ else:
+ raise _TrajectoryMergeError("unresolved embedded subagent trajectory_id")
+ if embedded:
+ trajectory["subagent_trajectories"] = embedded
+ else:
+ trajectory.pop("subagent_trajectories", None)
+ return _validated_trajectory_dict(trajectory)
+
+
+def _materialize_trajectory_file(
+ agent_dir: Path,
+ reference: Any,
+ *,
+ state: _TrajectoryReferenceState | None = None,
+ depth: int = 0,
+ count_against_reference_budget: bool = True,
+) -> tuple[dict[str, Any], str]:
+ if depth > _MAX_TRAJECTORY_REFERENCE_DEPTH:
+ raise _TrajectoryMergeError("trajectory reference depth exceeded")
+ state = state or _TrajectoryReferenceState()
+ _relative, key = _normalized_trajectory_reference(reference)
+ cache_key = _trajectory_reference_cache_key(agent_dir, key)
+ if cached := state.materialized_cache.get(cache_key):
+ return copy.deepcopy(cached), key
+ if cache_key in state.active:
+ raise _TrajectoryMergeError("trajectory reference cycle")
+ state.active.add(cache_key)
+ try:
+ trajectory, _key = _read_referenced_trajectory(
+ agent_dir,
+ reference,
+ state,
+ count_against_reference_budget=count_against_reference_budget,
+ )
+ continued_ref = trajectory.pop("continued_trajectory_ref", None)
+ combined_continuation = False
+ if continued_ref:
+ continuation, _continuation_key = _materialize_trajectory_file(
+ agent_dir,
+ continued_ref,
+ state=state,
+ depth=depth + 1,
+ )
+ trajectory = _combine_continuation_trajectories(trajectory, continuation, state=state)
+ combined_continuation = True
+ trajectory = _resolve_subagent_trajectory_refs(
+ trajectory,
+ agent_dir=agent_dir,
+ state=state,
+ depth=depth,
+ )
+ if combined_continuation:
+ state.generated_continuation_fingerprints[trajectory["trajectory_id"]] = _raw_trajectory_sha256(trajectory)
+ state.materialized_cache[cache_key] = trajectory
+ return copy.deepcopy(trajectory), key
+ finally:
+ state.active.discard(cache_key)
+
+
+def _trial_resumes_step_trajectories(trial_root: Path) -> bool:
+ """Read Harbor 0.22's serialized agent resume flag without coercion."""
+ result = _read_json(trial_root / "result.json")
+ if not isinstance(result, dict):
+ return False
+ config = result.get("config")
+ if not isinstance(config, dict):
+ return False
+ agent = config.get("agent")
+ return isinstance(agent, dict) and agent.get("resume_trajectory") is True
+
+
+def _valid_step_result_names(value: Any, *, max_count: int | None = None) -> list[str] | None:
+ """Return structurally authoritative Harbor step names, or ``None``."""
+ if not isinstance(value, list) or not value or (max_count is not None and len(value) > max_count):
+ return None
+ names: list[str] = []
+ seen_names: set[str] = set()
+ for step in value:
+ if not isinstance(step, dict):
+ return None
+ step_name = step.get("step_name")
+ if not isinstance(step_name, str) or not step_name.strip() or step_name in seen_names:
+ return None
+ names.append(step_name)
+ seen_names.add(step_name)
+ return names
+
+
+def _is_serialized_harbor_trial_result(result: dict[str, Any]) -> bool:
+ """Recognize Harbor's complete TrialResult envelope, not legacy fragments."""
+ config = result.get("config")
+ return (
+ isinstance(result.get("trial_uri"), str)
+ and isinstance(result.get("task_checksum"), str)
+ and isinstance(result.get("agent_info"), dict)
+ and isinstance(config, dict)
+ and isinstance(config.get("task"), dict)
+ )
+
+
+def _expected_step_trajectory_names(trial_root: Path) -> list[str] | None:
+ result = _read_json(trial_root / "result.json")
+ if not isinstance(result, dict):
+ return None
+ return _valid_step_result_names(
+ result.get("step_results"),
+ max_count=_MAX_TRAJECTORY_STEP_DIRECTORIES,
+ )
+
+
+def _merged_step_trajectory(trial_root: Path) -> dict[str, Any] | None:
+ """Merge a complete Harbor multi-step ATIF set or fail closed."""
+ try:
+ (trial_root / "agent" / "trajectory.json").lstat()
+ except FileNotFoundError:
+ pass
+ except OSError:
+ return None
+ else:
+ # Root and native step trajectories are contradictory authorities.
+ return None
+ expected_names = _expected_step_trajectory_names(trial_root)
+ paths = _ordered_step_trajectory_paths(trial_root)
+ discovered_names = [path.parent.parent.name for path in paths]
+ if expected_names is None or discovered_names != expected_names:
+ return None
+
+ try:
+ trajectories: list[tuple[str, dict[str, Any]]] = []
+ reference_state = _TrajectoryReferenceState()
+ for step_name, path in zip(expected_names, paths, strict=True):
+ trajectory, _reference_key = _materialize_trajectory_file(
+ path.parent,
+ path.name,
+ state=reference_state,
+ count_against_reference_budget=False,
+ )
+ trajectories.append((step_name, trajectory))
+ if not trajectories:
+ return None
+
+ agent_identity = _agent_identity(trajectories[0][1].get("agent"))
+ if agent_identity is None or any(
+ _agent_identity(trajectory.get("agent")) != agent_identity for _, trajectory in trajectories[1:]
+ ):
+ raise _TrajectoryMergeError("multi-step agent mismatch")
+
+ merged_steps: list[dict[str, Any]] = []
+ scoped_trajectories: list[tuple[str, dict[str, Any]]] = []
+ resume_trajectory = _trial_resumes_step_trajectories(trial_root)
+ continuation_flags: list[bool] = []
+ previous_trajectory: dict[str, Any] | None = None
+ for source_index, (step_name, trajectory) in enumerate(trajectories):
+ is_continuation = bool(
+ resume_trajectory
+ and previous_trajectory is not None
+ and _is_trusted_materialized_cumulative_continuation(
+ previous_trajectory,
+ trajectory,
+ state=reference_state,
+ )
+ )
+ continuation_flags.append(is_continuation)
+ scoped_trajectory = _remap_parent_subagent_scope(
+ trajectory,
+ namespace=f"harbor-step:{source_index}",
+ )
+ scoped_trajectories.append((step_name, scoped_trajectory))
+ steps = _trajectory_dict_steps(scoped_trajectory)
+ if is_continuation:
+ copied_prefix_length = _copied_context_prefix_length(steps)
+ previous_length = len(_trajectory_dict_steps(previous_trajectory)) if previous_trajectory else 0
+ steps = steps[copied_prefix_length or previous_length :]
+ for step in steps:
+ merged_step = copy.deepcopy(step)
+ original_step_id = merged_step.get("step_id")
+ merged_step["step_id"] = len(merged_steps) + 1
+ extra = merged_step.get("extra")
+ if not isinstance(extra, dict):
+ extra = {}
+ extra["harbor_step_name"] = step_name
+ if original_step_id not in (None, ""):
+ extra["harbor_original_step_id"] = original_step_id
+ else:
+ extra.pop("harbor_original_step_id", None)
+ merged_step["extra"] = extra
+ merged_steps.append(merged_step)
+ previous_trajectory = trajectory
+ if not merged_steps:
+ raise _TrajectoryMergeError("empty merged trajectory")
+
+ source_provenance = [
+ {
+ "step_name": step_name,
+ "schema_version": trajectory.get("schema_version"),
+ "session_id": trajectory.get("session_id"),
+ "trajectory_id": trajectory.get("trajectory_id"),
+ "root_extra": copy.deepcopy(trajectory.get("extra")),
+ "agent_extra": copy.deepcopy(
+ agent.get("extra") if isinstance((agent := trajectory.get("agent")), dict) else None
+ ),
+ "final_metrics_extra": copy.deepcopy(
+ final_metrics.get("extra")
+ if isinstance((final_metrics := trajectory.get("final_metrics")), dict)
+ else None
+ ),
+ }
+ for step_name, trajectory in trajectories
+ ]
+ source_identity = [
+ {
+ "source_index": source_index,
+ "step_name": redact_sensitive_text(step_name),
+ "is_continuation": continuation_flags[source_index],
+ "content_sha256": _canonical_trajectory_sha256(trajectory),
+ }
+ for source_index, (step_name, trajectory) in enumerate(trajectories)
+ ]
+ digest = hashlib.sha256(
+ json.dumps(source_identity, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
+ ).hexdigest()[:20]
+ merged: dict[str, Any] = {
+ "schema_version": "ATIF-v1.7",
+ "session_id": f"skillevaluator-multistep-{digest}",
+ "trajectory_id": f"skillevaluator-multistep-{digest}",
+ "agent": copy.deepcopy(trajectories[-1][1]["agent"]),
+ "steps": merged_steps,
+ "final_metrics": _merge_trajectory_final_metrics(
+ [trajectory for _, trajectory in trajectories],
+ continuation_flags=continuation_flags,
+ total_steps=len(merged_steps),
+ ),
+ "extra": {
+ "harbor_multi_step": {
+ "step_count": len(expected_names),
+ "step_names": expected_names,
+ "source_trajectories": source_provenance,
+ }
+ },
+ }
+ notes = _combined_notes(
+ *(
+ f"[{step_name}] {note}"
+ for step_name, trajectory in trajectories
+ if isinstance((note := trajectory.get("notes")), str) and note
+ )
+ )
+ if notes:
+ merged["notes"] = notes
+ subagents = _merged_embedded_subagents(
+ *(trajectory.get("subagent_trajectories") for _, trajectory in scoped_trajectories)
+ )
+ if subagents:
+ merged["subagent_trajectories"] = subagents
+ return _validated_trajectory_dict(merged)
+ except (OSError, SecurePathError, _TrajectoryMergeError, RecursionError):
+ logger.debug("Harbor multi-step trajectory could not be materialized", exc_info=True)
+ return None
def _merge_trajectory_final_metrics(
trajectories: list[dict[str, Any]],
*,
+ continuation_flags: list[bool],
total_steps: int,
) -> dict[str, Any]:
metrics: dict[str, Any] = {}
- for key in ("total_prompt_tokens", "total_completion_tokens", "total_cached_tokens"):
- values = [
- final_metrics.get(key)
- for trajectory in trajectories
- if isinstance(final_metrics := trajectory.get("final_metrics"), dict)
- ]
- numeric = [value for value in values if isinstance(value, int | float) and not isinstance(value, bool)]
- if numeric:
- metrics[key] = sum(numeric)
+ for key in (
+ "total_prompt_tokens",
+ "total_completion_tokens",
+ "total_cached_tokens",
+ "total_cost_usd",
+ ):
+ value = _sum_trajectory_metric_segments(
+ trajectories,
+ continuation_flags=continuation_flags,
+ key=key,
+ )
+ if value is not None:
+ metrics[key] = value
metrics["total_steps"] = total_steps
- last_final_metrics = next(
- (
- trajectory.get("final_metrics")
- for trajectory in reversed(trajectories)
- if isinstance(trajectory.get("final_metrics"), dict)
- ),
- {},
- )
+ terminal_final_metrics = trajectories[-1].get("final_metrics") if trajectories else None
+ last_final_metrics = terminal_final_metrics if isinstance(terminal_final_metrics, dict) else {}
last_extra = last_final_metrics.get("extra") if isinstance(last_final_metrics, dict) else None
- extra = copy.deepcopy(last_extra) if isinstance(last_extra, dict) else {}
+ extra = {
+ key: copy.deepcopy(last_extra[key])
+ for key in ("finish_reason",)
+ if isinstance(last_extra, dict) and key in last_extra
+ }
extra_token_keys = sorted(
{
str(key)
@@ -1315,25 +2824,62 @@ def _merge_trajectory_final_metrics(
if _is_aggregate_extra_token_key(str(key))
}
)
+ for key in list(extra):
+ if _is_aggregate_extra_token_key(str(key)):
+ extra.pop(key, None)
for key in extra_token_keys:
- numeric_values: list[int | float] = []
- for trajectory in trajectories:
- final_metrics = trajectory.get("final_metrics")
- if not isinstance(final_metrics, dict):
- continue
- step_extra = final_metrics.get("extra")
- if not isinstance(step_extra, dict):
- continue
- value = step_extra.get(key)
- if isinstance(value, int | float) and not isinstance(value, bool):
- numeric_values.append(value)
- if numeric_values:
- extra[key] = sum(numeric_values)
+ value = _sum_trajectory_metric_segments(
+ trajectories,
+ continuation_flags=continuation_flags,
+ key=key,
+ from_extra=True,
+ )
+ if value is not None:
+ extra[key] = value
extra["harbor_multi_step"] = True
metrics["extra"] = extra
return metrics
+def _sum_trajectory_metric_segments(
+ trajectories: list[dict[str, Any]],
+ *,
+ continuation_flags: list[bool],
+ key: str,
+ from_extra: bool = False,
+) -> int | float | None:
+ """Sum independent fragments while taking the latest cumulative value per resume chain."""
+ total: int | float = 0
+ segment_value: int | float | None = None
+ segment_known = False
+ for index, trajectory in enumerate(trajectories):
+ if index == 0 or not continuation_flags[index]:
+ if index > 0:
+ if not segment_known or segment_value is None:
+ return None
+ total += segment_value
+ segment_value = None
+ segment_known = False
+
+ final_metrics = trajectory.get("final_metrics")
+ source = final_metrics.get("extra") if from_extra and isinstance(final_metrics, dict) else final_metrics
+ value = source.get(key) if isinstance(source, dict) else None
+ try:
+ is_finite_number = isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value)
+ except OverflowError:
+ is_finite_number = False
+ if is_finite_number:
+ segment_value = value
+ segment_known = True
+ else:
+ segment_value = None
+ segment_known = False
+
+ if not segment_known or segment_value is None:
+ return None
+ return total + segment_value
+
+
def _merge_reward_sidecars(data: dict[str, Any], verifier_dir: Path) -> None:
"""Merge SkillEvaluator-rich sidecars back into Harbor's numeric-only reward payload."""
skill_evaluator_reward = _read_json(verifier_dir / "skill_evaluator_reward.json")
@@ -1361,6 +2907,14 @@ def _merge_reward_sidecars(data: dict[str, Any], verifier_dir: Path) -> None:
if not isinstance(custom_reward, dict):
return
+ if custom_metric_contract_error(custom_reward):
+ data["evaluation_status"] = "failed"
+ data["evaluation_errors"] = _merge_bounded_evaluation_errors(
+ {"collector": UNSAFE_CUSTOM_METRICS_REASON},
+ data.get("evaluation_errors"),
+ )
+ return
+
custom_metrics = extract_custom_metrics(custom_reward)
if custom_metrics:
existing = data.get("custom_metrics")
@@ -1378,7 +2932,11 @@ def _merge_reward_sidecars(data: dict[str, Any], verifier_dir: Path) -> None:
details[metric] = detail
if details:
data["details"] = details
- data["custom_details"] = custom_details
+ safe_custom_details = {
+ str(metric): detail for metric, detail in custom_details.items() if str(metric) in custom_metrics
+ }
+ if safe_custom_details:
+ data["custom_details"] = safe_custom_details
for key in ("entry_id", "error"):
value = custom_reward.get(key)
@@ -1440,9 +2998,9 @@ def _standard_reward_metrics(
if isinstance(nested_metrics, dict):
metric_names.update(str(name) for name in nested_metrics)
declared_metric_set = str(rewards.get("metric_set") or rewards.get("metric_set_version") or "")
- if declared_metric_set not in {DEFAULT_METRIC_SET, LEGACY_METRIC_SET} and not metric_names.intersection(
- DEFAULT_METRICS
- ):
+ if declared_metric_set and declared_metric_set not in {DEFAULT_METRIC_SET, LEGACY_METRIC_SET}:
+ return ()
+ if not declared_metric_set and not metric_names.intersection(DEFAULT_METRICS):
return ()
_, expected_metrics = metric_set_for_reward(rewards)
@@ -1466,6 +3024,69 @@ def _physical_steps_layout_present(trial_root: Path) -> bool:
return True
+def _reward_claims_metric(rewards: dict[str, Any], metric: str) -> bool:
+ if metric in rewards:
+ return True
+ nested = rewards.get("metrics")
+ return isinstance(nested, dict) and metric in nested
+
+
+def _standard_aggregate_matches_harbor_strategy(
+ root_rewards: dict[str, Any],
+ step_results: list[dict[str, Any]],
+ root_metrics: tuple[str, ...],
+) -> bool:
+ """Recognize Harbor's FINAL or missing-as-zero MEAN aggregate semantics."""
+ root_values = {metric: metric_value(root_rewards, metric) for metric in root_metrics}
+ if any(value is None for value in root_values.values()):
+ return False
+
+ final_rewards: dict[str, Any] | None = None
+ if step_results:
+ final_verifier = step_results[-1].get("verifier_result")
+ candidate = final_verifier.get("rewards") if isinstance(final_verifier, dict) else None
+ if isinstance(candidate, dict):
+ final_rewards = candidate
+ if final_rewards is not None and all(
+ (value := metric_value(final_rewards, metric)) is not None
+ and math.isclose(value, root_values[metric], rel_tol=1e-9, abs_tol=1e-9)
+ for metric in root_metrics
+ ):
+ return True
+
+ verifier_rewards: list[dict[str, Any]] = []
+ for step in step_results:
+ verifier_result = step.get("verifier_result")
+ if verifier_result is None:
+ continue
+ if not isinstance(verifier_result, dict):
+ return False
+ rewards = verifier_result.get("rewards")
+ if rewards is None:
+ verifier_rewards.append({})
+ elif isinstance(rewards, dict):
+ verifier_rewards.append(rewards)
+ else:
+ return False
+ if not verifier_rewards:
+ return False
+
+ for metric in root_metrics:
+ values: list[float] = []
+ for rewards in verifier_rewards:
+ if not _reward_claims_metric(rewards, metric):
+ values.append(0.0)
+ continue
+ value = metric_value(rewards, metric)
+ if value is None:
+ return False
+ values.append(value)
+ mean = sum(values) / len(verifier_rewards)
+ if not math.isclose(mean, root_values[metric], rel_tol=1e-9, abs_tol=1e-9):
+ return False
+ return True
+
+
def _constituent_default_reward_failure(result: dict[str, Any], trial_root: Path | None = None) -> str:
"""Return a safe failure when a standard step reward cannot support its aggregate."""
root_verifier = result.get("verifier_result")
@@ -1498,8 +3119,17 @@ def _constituent_default_reward_failure(result: dict[str, Any], trial_root: Path
return "Authoritative verifier result has malformed constituent steps; it was not scored"
if not isinstance(step_results, list):
return ""
+ if _valid_step_result_names(step_results) is None:
+ return "Authoritative verifier result has malformed constituent steps; it was not scored"
+ if _is_serialized_harbor_trial_result(result) and not isinstance(root_rewards, dict):
+ return MISSING_MULTI_STEP_REWARD_REASON
root_metrics = _standard_reward_metrics(root_rewards) if isinstance(root_rewards, dict) else ()
+ if root_metrics and not _standard_aggregate_matches_harbor_strategy(root_rewards, step_results, root_metrics):
+ return (
+ "Constituent default rewards do not match Harbor's final or mean aggregate semantics; "
+ "the authoritative aggregate was not scored"
+ )
for index, step in enumerate(step_results, start=1):
if not isinstance(step, dict):
@@ -1534,7 +3164,7 @@ def _constituent_default_reward_failure(result: dict[str, Any], trial_root: Path
"error",
"failed",
}
- if failed_status or (root_metrics and (not isinstance(rewards, dict) or not rewards)):
+ if failed_status or (root_metrics and rewards is not None and not isinstance(rewards, dict)):
return (
f"Constituent default reward for step {step_name} is incomplete, non-finite, or failed; "
"the authoritative aggregate was not scored"
@@ -1548,7 +3178,10 @@ def _constituent_default_reward_failure(result: dict[str, Any], trial_root: Path
)
if not expected_metrics:
continue
- if all(metric_value(rewards, metric) is not None for metric in expected_metrics):
+ if all(
+ not _reward_claims_metric(rewards, metric) or metric_value(rewards, metric) is not None
+ for metric in expected_metrics
+ ):
continue
return (
@@ -1558,13 +3191,62 @@ def _constituent_default_reward_failure(result: dict[str, Any], trial_root: Path
return ""
+def _constituent_custom_metric_failure(result: dict[str, Any]) -> str:
+ """Validate custom metric bounds without reconstructing Harbor's root reward."""
+ reward_rows: list[dict[str, Any]] = []
+ root_verifier = result.get("verifier_result")
+ root_rewards = root_verifier.get("rewards") if isinstance(root_verifier, dict) else None
+ if isinstance(root_rewards, dict):
+ reward_rows.append(root_rewards)
+
+ step_results = result.get("step_results")
+ if isinstance(step_results, list):
+ for step in step_results:
+ if not isinstance(step, dict):
+ continue
+ verifier_result = step.get("verifier_result")
+ rewards = verifier_result.get("rewards") if isinstance(verifier_result, dict) else None
+ if isinstance(rewards, dict):
+ reward_rows.append(rewards)
+
+ custom_names: set[str] = set()
+ for rewards in reward_rows:
+ for raw_name, raw_value in rewards.items():
+ name = str(raw_name)
+ if isinstance(raw_value, int | float) and not isinstance(raw_value, bool):
+ continue
+ if name == "metrics" and isinstance(raw_value, dict):
+ nested_standard_metrics_are_valid = all(
+ str(metric) in DEFAULT_METRICS
+ and score_value(value.get("score") if isinstance(value, dict) else value) is not None
+ for metric, value in raw_value.items()
+ )
+ if nested_standard_metrics_are_valid:
+ continue
+ if name in RESERVED_METRIC_NAMES and name not in {"custom_metrics", "metrics"}:
+ continue
+ if name.startswith("_"):
+ continue
+ return MALFORMED_HARBOR_REWARD_REASON
+ if custom_metric_contract_error(rewards):
+ return UNSAFE_CUSTOM_METRICS_REASON
+ custom_names.update(extract_custom_metrics(rewards))
+ if len(custom_names) > MAX_CUSTOM_METRICS:
+ return UNSAFE_CUSTOM_METRICS_REASON
+ return ""
+
+
def _merge_constituent_default_reward_failure(
data: dict[str, Any],
result: dict[str, Any],
trial_root: Path | None = None,
) -> None:
- """Make an aggregate unscoreable when one of its standard constituents is invalid."""
- reason = _constituent_default_reward_failure(result, trial_root)
+ """Make an aggregate unscoreable when one of its constituents is invalid."""
+ reasons = (
+ _constituent_default_reward_failure(result, trial_root),
+ _constituent_custom_metric_failure(result),
+ )
+ reason = "; ".join(item for item in reasons if item)
if not reason:
return
data["evaluation_status"] = "failed"
@@ -1574,7 +3256,10 @@ def _merge_constituent_default_reward_failure(
)
-def _extract_rewards(job_dir: Path) -> list[dict[str, Any]]:
+def _extract_rewards(
+ job_dir: Path,
+ case_id_by_task_selector: dict[str, str] | None = None,
+) -> list[dict[str, Any]]:
"""Extract reward.json from all trials in a job directory."""
rewards: list[dict[str, Any]] = []
scored_trial_roots: set[Path] = set()
@@ -1602,13 +3287,11 @@ def _extract_rewards(job_dir: Path) -> list[dict[str, Any]]:
data["_trial_name"] = trial_name
data["_trial_root_name"] = trial_dir.name
data["_started_at"] = result.get("started_at")
- if not data.get("entry_id"):
- entry_id = _entry_id_from_harbor_result(result)
- if entry_id:
- data["entry_id"] = entry_id
+ _apply_harbor_result_case_identity(data, result, case_id_by_task_selector)
traj_file = _reward_trajectory_path(trial_dir, None)
if traj_file.exists():
data["_has_trajectory"] = True
+ data = _fail_closed_invalid_reward_numbers(data)
rewards.append(data)
authoritative_trial_roots.add(trial_dir)
@@ -1637,18 +3320,18 @@ def _extract_rewards(job_dir: Path) -> list[dict[str, Any]]:
if step_name:
data["_step_name"] = step_name
result_file = trial_dir / "result.json"
+ result: dict[str, Any] = {}
if result_file.exists():
- result = _read_json(result_file)
- if isinstance(result, dict):
+ loaded_result = _read_json(result_file)
+ if isinstance(loaded_result, dict):
+ result = loaded_result
_merge_constituent_default_reward_failure(data, result, trial_dir)
data["_started_at"] = result.get("started_at")
- if not data.get("entry_id"):
- entry_id = _entry_id_from_harbor_result(result)
- if entry_id:
- data["entry_id"] = entry_id
+ _apply_harbor_result_case_identity(data, result, case_id_by_task_selector)
traj_file = _reward_trajectory_path(trial_dir, step_name)
if traj_file.exists():
data["_has_trajectory"] = True
+ data = _fail_closed_invalid_reward_numbers(data)
rewards.append(data)
scored_trial_roots.add(trial_dir)
except OSError as e:
@@ -1666,6 +3349,37 @@ def _extract_rewards(job_dir: Path) -> list[dict[str, Any]]:
result = _read_json(result_file)
if not isinstance(result, dict):
continue
+ # Older Harbor fragments can contain only embedded step verifier rows,
+ # with no root aggregate and no physical reward sidecars. Preserve the
+ # rows as separate diagnostics so row-specific metric-set semantics and
+ # logical-overall weighting survive collection and report reloads.
+ step_names = (
+ None if _is_serialized_harbor_trial_result(result) else _valid_step_result_names(result.get("step_results"))
+ )
+ embedded_rows: list[tuple[str, dict[str, Any]]] = []
+ if step_names is not None:
+ for step_name, step in zip(step_names, result["step_results"], strict=True):
+ step_rewards = _harbor_result_rewards({"step_results": [step]})
+ data = (
+ _reward_from_harbor_result({"verifier_result": {"rewards": step_rewards}}) if step_rewards else None
+ )
+ if data:
+ embedded_rows.append((step_name, data))
+ if embedded_rows:
+ for step_name, data in embedded_rows:
+ _merge_constituent_default_reward_failure(data, result, trial_dir)
+ _merge_trial_evaluation_failures(data, trial_dir)
+ trial_name = str(result.get("trial_name") or trial_dir.name)
+ data["_trial_name"] = trial_name
+ data["_trial_root_name"] = trial_dir.name
+ data["_step_name"] = step_name
+ data["_started_at"] = result.get("started_at")
+ _apply_harbor_result_case_identity(data, result, case_id_by_task_selector)
+ traj_file = _reward_trajectory_path(trial_dir, step_name)
+ if traj_file.exists():
+ data["_has_trajectory"] = True
+ rewards.append(_fail_closed_invalid_reward_numbers(data))
+ continue
data = _reward_from_harbor_result(result)
if not data:
continue
@@ -1675,28 +3389,180 @@ def _extract_rewards(job_dir: Path) -> list[dict[str, Any]]:
data["_trial_name"] = trial_name
data["_trial_root_name"] = trial_dir.name
data["_started_at"] = result.get("started_at")
- if not data.get("entry_id"):
- entry_id = _entry_id_from_harbor_result(result)
- if entry_id:
- data["entry_id"] = entry_id
+ _apply_harbor_result_case_identity(data, result, case_id_by_task_selector)
traj_file = _reward_trajectory_path(trial_dir, None)
if traj_file.exists():
data["_has_trajectory"] = True
+ data = _fail_closed_invalid_reward_numbers(data)
rewards.append(data)
return rewards
+def _finite_reward_number(value: Any) -> float | None:
+ """Convert a Harbor numeric reward without propagating overflow or non-finite values."""
+ if not isinstance(value, int | float) or isinstance(value, bool):
+ return None
+ try:
+ numeric = float(value)
+ except (OverflowError, ValueError):
+ return None
+ return numeric if math.isfinite(numeric) else None
+
+
+class _RewardStructureLimitError(ValueError):
+ """Raised when a reward cannot be copied safely for generated output."""
+
+
+def _normalized_reward_numbers(
+ value: Any,
+ *,
+ _depth: int = 1,
+ _node_budget: list[int] | None = None,
+ _max_nodes: int = COLLECTED_REWARD_JSON_MAX_NODES,
+) -> tuple[Any, bool]:
+ """Return JSON-safe reward data and whether an invalid number was replaced."""
+ node_budget = _node_budget if _node_budget is not None else [0]
+ node_budget[0] += 1
+ if node_budget[0] > _max_nodes:
+ raise _RewardStructureLimitError("reward node count exceeds limit")
+ if isinstance(value, bool):
+ return value, False
+ if isinstance(value, int):
+ if abs(value) > _MAX_JSON_SAFE_INTEGER:
+ return None, True
+ return value, False
+ if isinstance(value, float):
+ if _finite_reward_number(value) is None:
+ return None, True
+ return value, False
+ if isinstance(value, dict):
+ if _depth > REWARD_JSON_MAX_DEPTH:
+ raise _RewardStructureLimitError("reward depth exceeds limit")
+ invalid = False
+ normalized: dict[str, Any] = {}
+ for key, item in value.items():
+ normalized_item, item_invalid = _normalized_reward_numbers(
+ item,
+ _depth=_depth + 1,
+ _node_budget=node_budget,
+ _max_nodes=_max_nodes,
+ )
+ normalized[str(key)] = normalized_item
+ invalid = invalid or item_invalid
+ return normalized, invalid
+ if isinstance(value, list):
+ if _depth > REWARD_JSON_MAX_DEPTH:
+ raise _RewardStructureLimitError("reward depth exceeds limit")
+ invalid = False
+ normalized_items: list[Any] = []
+ for item in value:
+ normalized_item, item_invalid = _normalized_reward_numbers(
+ item,
+ _depth=_depth + 1,
+ _node_budget=node_budget,
+ _max_nodes=_max_nodes,
+ )
+ normalized_items.append(normalized_item)
+ invalid = invalid or item_invalid
+ return normalized_items, invalid
+ return value, False
+
+
+def _structural_limit_reward(data: dict[str, Any]) -> dict[str, Any]:
+ """Keep only shallow identifiers when a reward payload is unsafe to traverse."""
+ safe: dict[str, Any] = {}
+ for key in (
+ "_trial_name",
+ "_trial_root_name",
+ "_step_name",
+ "_started_at",
+ "entry_id",
+ "trial_id",
+ "agent",
+ "model",
+ "model_source",
+ ):
+ value = data.get(key)
+ if isinstance(value, str):
+ safe_value = _safe_diagnostic_text(value, max_len=512)
+ if safe_value:
+ safe[key] = safe_value
+ if isinstance(data.get("_has_trajectory"), bool):
+ safe["_has_trajectory"] = data["_has_trajectory"]
+ safe["evaluation_status"] = "failed"
+ safe["evaluation_errors"] = {"collector": UNSAFE_REWARD_STRUCTURE_REASON}
+ return safe
+
+
+def _fail_closed_invalid_reward_numbers(
+ data: dict[str, Any],
+ *,
+ max_nodes: int = COLLECTED_REWARD_JSON_MAX_NODES,
+ max_bytes: int = COLLECTED_REWARD_JSON_MAX_BYTES,
+) -> dict[str, Any]:
+ """Replace invalid numbers and attach a bounded diagnostic to the reward."""
+ try:
+ normalized, invalid = _normalized_reward_numbers(data, _max_nodes=max_nodes)
+ encoded = json.dumps(
+ normalized,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ).encode("utf-8")
+ if len(encoded) > max_bytes:
+ raise _RewardStructureLimitError("reward bytes exceed generated-artifact limit")
+ except (_RewardStructureLimitError, TypeError, ValueError, RecursionError, MemoryError):
+ return _structural_limit_reward(data)
+ if not isinstance(normalized, dict):
+ return data
+ if custom_metric_contract_error(normalized):
+ normalized["evaluation_status"] = "failed"
+ normalized["evaluation_errors"] = _merge_bounded_evaluation_errors(
+ {"collector": UNSAFE_CUSTOM_METRICS_REASON},
+ normalized.get("evaluation_errors"),
+ )
+ if invalid:
+ normalized["evaluation_status"] = "failed"
+ normalized["evaluation_errors"] = _merge_bounded_evaluation_errors(
+ {"collector": UNSCOREABLE_NUMERIC_REWARD_REASON},
+ normalized.get("evaluation_errors"),
+ )
+ return normalized
+
+
def _reward_from_harbor_result(result: dict[str, Any]) -> dict[str, Any] | None:
+ root_verifier = result.get("verifier_result")
+ root_rewards = root_verifier.get("rewards") if isinstance(root_verifier, dict) else None
+ if (
+ _is_serialized_harbor_trial_result(result)
+ and _valid_step_result_names(result.get("step_results")) is not None
+ and not isinstance(root_rewards, dict)
+ ):
+ return {
+ "evaluation_status": "failed",
+ "evaluation_errors": {"collector": MISSING_MULTI_STEP_REWARD_REASON},
+ "details": {"harbor_rewards": {}},
+ }
+
harbor_rewards = _harbor_result_rewards(result)
if not harbor_rewards:
return None
data: dict[str, Any] = {}
custom_metrics: dict[str, float] = {}
+ safe_harbor_rewards: dict[str, Any] = {}
+ invalid_numeric_reward = False
for key, value in harbor_rewards.items():
+ if key == _CUSTOM_METRIC_CONTRACT_MARKER:
+ continue
if not isinstance(value, int | float) or isinstance(value, bool):
+ safe_harbor_rewards[str(key)] = value
+ continue
+ score = _finite_reward_number(value)
+ safe_harbor_rewards[str(key)] = value if score is not None else None
+ if score is None:
+ invalid_numeric_reward = True
continue
- score = float(value)
if key in DEFAULT_METRICS:
data[key] = score
elif key == "overall":
@@ -1707,14 +3573,42 @@ def _reward_from_harbor_result(result: dict[str, Any]) -> dict[str, Any] | None:
else:
custom_metrics[key] = score
+ metric_set = harbor_rewards.get("metric_set") or harbor_rewards.get("metric_set_version")
+ if isinstance(metric_set, str) and metric_set:
+ data["metric_set"] = metric_set
+
+ if invalid_numeric_reward:
+ data["evaluation_status"] = "failed"
+ data["evaluation_errors"] = {"collector": UNSCOREABLE_NUMERIC_REWARD_REASON}
+ if harbor_rewards.get(_CUSTOM_METRIC_CONTRACT_MARKER):
+ data["evaluation_status"] = "failed"
+ data["evaluation_errors"] = {"collector": UNSAFE_CUSTOM_METRICS_REASON}
if not any(not k.startswith("_") for k in data) and not custom_metrics:
return None
if custom_metrics:
data["custom_metrics"] = custom_metrics
- data["details"] = {"harbor_rewards": harbor_rewards}
+ data["details"] = {"harbor_rewards": safe_harbor_rewards}
return data
+def _average_multistep_custom_metrics(rewards: list[dict[str, Any]]) -> dict[str, float]:
+ """Mirror Harbor 0.22 MEAN aggregation by zero-filling missing custom keys."""
+ if not rewards:
+ return {}
+ names: set[str] = set()
+ extracted_rows: list[dict[str, float]] = []
+ for reward in rewards:
+ if reason := custom_metric_contract_error(reward):
+ raise CustomMetricContractError(reason)
+ extracted = extract_custom_metrics(reward)
+ names.update(extracted)
+ if len(names) > MAX_CUSTOM_METRICS:
+ raise CustomMetricContractError("Custom metric union exceeds the per condition publication limit")
+ extracted_rows.append(extracted)
+ denominator = len(extracted_rows)
+ return {name: round(sum(row.get(name, 0.0) for row in extracted_rows) / denominator, 4) for name in sorted(names)}
+
+
def _harbor_result_rewards(result: dict[str, Any]) -> dict[str, Any] | None:
verifier_result = result.get("verifier_result")
if isinstance(verifier_result, dict):
@@ -1739,39 +3633,57 @@ def _harbor_result_rewards(result: dict[str, Any]) -> dict[str, Any] | None:
aggregated: dict[str, Any] = {}
for key in sorted({str(key) for rewards in step_reward_rows for key in rewards}):
- values = [
- float(rewards[key])
+ raw_values = [
+ rewards[key]
for rewards in step_reward_rows
if isinstance(rewards.get(key), int | float) and not isinstance(rewards.get(key), bool)
]
- if values:
+ values = [_finite_reward_number(value) for value in raw_values]
+ if any(value is None for value in values):
+ # Preserve one invalid numeric long enough for the shared reward
+ # normalizer to mark the whole artifact unscoreable and replace it
+ # with strict-JSON ``null``. Silently dropping only the bad step
+ # would let a partial average pass.
+ aggregated[key] = next(
+ raw_value for raw_value, value in zip(raw_values, values, strict=True) if value is None
+ )
+ elif values:
aggregated[key] = sum(values) / len(values)
- aggregated.update(average_custom_metrics(step_reward_rows))
-
- standard_rows = [rewards for rewards in step_reward_rows if _standard_reward_metrics(rewards)]
- active_metrics: tuple[str, ...] = ()
- if any(_standard_reward_metrics(rewards) == DEFAULT_METRICS for rewards in standard_rows):
- active_metrics = DEFAULT_METRICS
- elif standard_rows:
- active_metrics = LEGACY_METRICS
- for metric in active_metrics:
- values = [value for rewards in standard_rows if (value := metric_value(rewards, metric)) is not None]
- if values:
- aggregated[metric] = sum(values) / len(values)
- return aggregated or None
+ try:
+ aggregated.update(_average_multistep_custom_metrics(step_reward_rows))
+ except CustomMetricContractError:
+ return {
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ _CUSTOM_METRIC_CONTRACT_MARKER: True,
+ }
+ # Classify each row before accepting canonical metric names. An explicitly
+ # custom-only step may carry arbitrary keys, including reserved names, but
+ # those names must never be reclassified as SkillEvaluator-owned scores.
+ standard_scores, metric_set, _active_metrics = average_metrics(step_reward_rows)
+ has_standard_contract = any(_standard_reward_metrics(rewards) for rewards in step_reward_rows)
+ if not has_standard_contract and any(
+ _finite_reward_number(aggregated.get(name)) is not None for name in ("overall", "reward")
+ ):
+ metric_set = CUSTOM_ONLY_METRIC_SET
+ for metric in DEFAULT_METRICS:
+ aggregated.pop(metric, None)
+ aggregated.update(standard_scores)
+ aggregated["metric_set"] = metric_set
+ if (logical_overall := _average_overall(step_reward_rows)) is not None:
+ aggregated["overall"] = logical_overall
+ return aggregated or None
-def _entry_id_from_harbor_result(result: dict[str, Any]) -> str:
- task_name = result.get("task_name")
- if isinstance(task_name, str) and task_name.strip():
- return task_name.strip().rsplit("/", 1)[-1]
+def _task_selector_from_harbor_result(result: dict[str, Any]) -> str:
+ """Return one consistent staged selector, never ``[task].name``."""
+ task_paths: list[str] = []
task_id = result.get("task_id")
if isinstance(task_id, dict):
task_path = task_id.get("path")
if isinstance(task_path, str) and task_path.strip():
- return Path(task_path).name
+ task_paths.append(task_path.strip())
config = result.get("config")
if isinstance(config, dict):
@@ -1779,15 +3691,251 @@ def _entry_id_from_harbor_result(result: dict[str, Any]) -> str:
if isinstance(task, dict):
task_path = task.get("path")
if isinstance(task_path, str) and task_path.strip():
- return Path(task_path).name
+ task_paths.append(task_path.strip())
+
+ if not task_paths or any(task_path != task_paths[0] for task_path in task_paths):
+ return ""
+ return Path(task_paths[0]).name
+
+
+def _entry_id_from_harbor_result(
+ result: dict[str, Any],
+ case_id_by_task_selector: dict[str, str] | None = None,
+) -> str:
+ if case_id_by_task_selector is None:
+ # Preserve the legacy collector contract for direct callers and older
+ # artifacts: Harbor's task_name is the logical/display identity they
+ # supplied. New runner paths pass an explicit trusted selector map and
+ # never use this authored field as logical truth.
+ task_name = result.get("task_name")
+ if isinstance(task_name, str) and task_name.strip():
+ return task_name.strip().rsplit("/", 1)[-1]
+
+ selector = _task_selector_from_harbor_result(result)
+ if selector:
+ return case_id_by_task_selector.get(selector, "") if case_id_by_task_selector is not None else selector
+
+ # A trusted mapping deliberately keeps authored ``[task].name`` separate
+ # from logical identity. Harbor 0.22 normally persists ``task_id.path``;
+ # if it is absent, fail coverage instead of guessing from a display name.
+ if case_id_by_task_selector is not None:
+ return ""
return ""
+def _positive_attempt_ordinal(value: object) -> int | None:
+ """Return a positive integer attempt ordinal without accepting booleans."""
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+ return value
+ if isinstance(value, str) and value.isascii() and value.isdigit():
+ ordinal = int(value)
+ return ordinal if ordinal > 0 else None
+ return None
+
+
+def _structural_attempt_ordinal(trial_root_name: object, task_selector: str) -> int | None:
+ """Read an ordinal only from runner-owned or exact legacy selector structure."""
+ if not isinstance(trial_root_name, str) or not trial_root_name or not task_selector:
+ return None
+
+ # stop_on_pass aggregates are named ``--attemptNNN__``.
+ # Consume the complete trusted selector before reading the runner-owned
+ # suffix so attempt-like text inside any authored identity is inert.
+ aggregate_marker = f"-{task_selector}-attempt"
+ marker_index = trial_root_name.rfind(aggregate_marker)
+ if marker_index >= 0:
+ tail = trial_root_name[marker_index + len(aggregate_marker) :]
+ ordinal_text, separator, child_name = tail.partition("__")
+ if separator and child_name:
+ return _positive_attempt_ordinal(ordinal_text)
+
+ # Long aggregate names carry the anchored runner attempt behind an exact
+ # digest suffix because the complete selector marker may not fit within a
+ # portable filesystem component. Harbor 0.22 native trial names use a
+ # seven-character ShortUUID and cannot produce this aggregate shape.
+ truncated_match = re.fullmatch(
+ rf".+{re.escape(TRUNCATED_AGGREGATE_ATTEMPT_PREFIX)}0*(?P[1-9][0-9]*)__[0-9a-f]{{16}}",
+ trial_root_name,
+ flags=re.IGNORECASE,
+ )
+ if truncated_match:
+ return _positive_attempt_ordinal(truncated_match.group("ordinal"))
+
+ # Harbor 0.13-era local trials used ``_attemptNNN``. Matching the
+ # complete trusted selector keeps compatibility without guessing from an
+ # arbitrary occurrence of ``attempt`` in the selector or display name.
+ legacy_marker = f"{task_selector}_attempt"
+ if trial_root_name.startswith(legacy_marker):
+ return _positive_attempt_ordinal(trial_root_name[len(legacy_marker) :])
+ return None
+
+
+def _apply_harbor_result_case_identity(
+ data: dict[str, Any],
+ result: dict[str, Any],
+ case_id_by_task_selector: dict[str, str] | None,
+) -> None:
+ """Apply trusted staged identity, or retain legacy reward fallback."""
+ data.pop("_attempt_ordinal", None)
+ data.pop("_trusted_task_selector", None)
+ task_selector = _task_selector_from_harbor_result(result)
+ if task_selector:
+ data["_trusted_task_selector"] = task_selector
+ attempt_ordinal = _structural_attempt_ordinal(data.get("_trial_root_name"), task_selector)
+ if attempt_ordinal is not None:
+ data["_attempt_ordinal"] = attempt_ordinal
+ entry_id = _entry_id_from_harbor_result(result, case_id_by_task_selector)
+ if case_id_by_task_selector is not None:
+ if entry_id:
+ data["entry_id"] = entry_id
+ data.pop("_trusted_case_identity_unresolved", None)
+ else:
+ # A grader-authored identity is not authoritative. If Harbor's
+ # persisted selector is missing, unknown, or internally
+ # inconsistent, make the reward diagnostic-only so coverage fails
+ # instead of allowing it to impersonate an expected case.
+ data.pop("entry_id", None)
+ data["_trusted_case_identity_unresolved"] = True
+ elif entry_id and not data.get("entry_id"):
+ data["entry_id"] = entry_id
+
+
def _overall_score(reward: dict[str, Any]) -> float | None:
+ if reward.get("_logical_attempt_sentinel") is _LOGICAL_ATTEMPT_SENTINEL:
+ return _finite_reward_number(reward.get("_logical_overall"))
return overall_score(reward)
+def _sanitize_reward_metric_surfaces(reward: dict[str, Any]) -> dict[str, Any]:
+ """Omit unsafe metric names without creating redaction aliases."""
+ contract_failed = custom_metric_contract_error(reward) is not None
+ sanitized = dict(reward)
+
+ for field in ("custom_metrics", "metrics"):
+ raw_metrics = reward.get(field)
+ if not isinstance(raw_metrics, dict):
+ continue
+ sanitized_metrics: dict[str, Any] = {}
+ for raw_name, value in raw_metrics.items():
+ name = str(raw_name)
+ if name in RESERVED_METRIC_NAMES:
+ sanitized_metrics[name] = value
+ continue
+ candidate = value.get("score") if isinstance(value, dict) else value
+ if not contract_failed and custom_metric_name_is_publishable(name) and score_value(candidate) is not None:
+ sanitized_metrics[name] = value
+ sanitized[field] = sanitized_metrics
+
+ for raw_name, value in list(reward.items()):
+ name = str(raw_name)
+ if name in RESERVED_METRIC_NAMES or name.startswith("_"):
+ continue
+ if not custom_metric_name_is_publishable(name):
+ sanitized.pop(raw_name, None)
+ continue
+ candidate = value.get("score") if isinstance(value, dict) else value
+ if score_value(candidate) is None:
+ continue
+ if contract_failed:
+ sanitized.pop(raw_name, None)
+
+ details = reward.get("details")
+ if isinstance(details, dict):
+ sanitized["details"] = {
+ str(raw_name): detail
+ for raw_name, detail in details.items()
+ if str(raw_name) in RESERVED_METRIC_NAMES or custom_metric_name_is_publishable(str(raw_name))
+ }
+ safe_details = sanitized.get("details")
+ harbor_rewards = details.get("harbor_rewards") if isinstance(details, dict) else None
+ if isinstance(harbor_rewards, dict):
+ safe_harbor_rewards: dict[str, Any] = {}
+ for raw_name, value in harbor_rewards.items():
+ name = str(raw_name)
+ if name in {"custom_metrics", "metrics"} and isinstance(value, dict):
+ safe_harbor_rewards[name] = {
+ str(raw_metric): metric_value
+ for raw_metric, metric_value in value.items()
+ if str(raw_metric) in RESERVED_METRIC_NAMES
+ or (
+ not contract_failed
+ and custom_metric_name_is_publishable(str(raw_metric))
+ and score_value(metric_value.get("score") if isinstance(metric_value, dict) else metric_value)
+ is not None
+ )
+ }
+ continue
+ if (
+ name in RESERVED_METRIC_NAMES
+ or name in {"reward", "metric_set", "metric_set_version"}
+ or (not contract_failed and custom_metric_name_is_publishable(name))
+ ):
+ safe_harbor_rewards[name] = value
+ safe_details = dict(safe_details) if isinstance(safe_details, dict) else {}
+ safe_details["harbor_rewards"] = safe_harbor_rewards
+ sanitized["details"] = safe_details
+
+ custom_details = reward.get("custom_details")
+ if isinstance(custom_details, dict):
+ custom_metric_names = set(extract_custom_metrics(reward)) if not contract_failed else set()
+ safe_custom_details = {
+ str(raw_name): detail for raw_name, detail in custom_details.items() if str(raw_name) in custom_metric_names
+ }
+ if safe_custom_details:
+ sanitized["custom_details"] = safe_custom_details
+ else:
+ sanitized.pop("custom_details", None)
+ return sanitized
+
+
+def _reward_publication_projection_is_safe(reward: dict[str, Any]) -> bool:
+ """Check the same redacted envelope used by persisted reward artifacts."""
+ clean_reward = _sanitize_reward_metric_surfaces(
+ {key: value for key, value in reward.items() if not key.startswith("_")}
+ )
+ diagnostic_reward = (
+ str(clean_reward.get("evaluation_status") or "").casefold() in {"error", "failed"}
+ or overall_score(clean_reward) is None
+ )
+ max_str_len = REWARD_DIAGNOSTIC_STRING_MAX_CHARS if diagnostic_reward else None
+ try:
+ safe_reward = redact_sensitive_data(
+ clean_reward,
+ max_str_len=max_str_len,
+ )
+ _restore_custom_metric_scores(clean_reward, safe_reward, max_str_len=max_str_len)
+ _restore_custom_metric_details(clean_reward, safe_reward, max_str_len=max_str_len)
+ normalized, _invalid = _normalized_reward_numbers(
+ safe_reward,
+ _max_nodes=COLLECTED_REWARD_JSON_MAX_NODES,
+ )
+ _validate_generated_json_value(
+ normalized,
+ max_depth=REWARD_JSON_MAX_DEPTH,
+ max_nodes=COLLECTED_REWARD_JSON_MAX_NODES,
+ max_bytes=COLLECTED_REWARD_JSON_MAX_BYTES,
+ )
+ except (
+ _RewardStructureLimitError,
+ MemoryError,
+ RecursionError,
+ TypeError,
+ UnicodeError,
+ ValueError,
+ ):
+ return False
+ return isinstance(normalized, dict)
+
+
+def _mark_reward_collection_failure(reward: dict[str, Any], reason: str) -> None:
+ reward["evaluation_status"] = "failed"
+ reward["evaluation_errors"] = _merge_bounded_evaluation_errors(
+ {"collector": reason},
+ reward.get("evaluation_errors"),
+ )
+
+
def _partition_scoreable_rewards(
rewards: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
@@ -1796,26 +3944,48 @@ def _partition_scoreable_rewards(
failures: list[dict[str, str]] = []
failed_trials: set[str] = set()
for reward in rewards:
+ if not _reward_identity_is_publishable(reward):
+ _mark_reward_collection_failure(reward, UNSAFE_REWARD_IDENTITY_REASON)
+ elif custom_metric_contract_error(reward):
+ _mark_reward_collection_failure(reward, UNSAFE_CUSTOM_METRICS_REASON)
+ elif not _reward_publication_projection_is_safe(reward):
+ _mark_reward_collection_failure(reward, UNSAFE_REWARD_STRUCTURE_REASON)
evaluation_failed = str(reward.get("evaluation_status") or "").casefold() in {"error", "failed"}
if not evaluation_failed and overall_score(reward) is not None:
scoreable.append(reward)
continue
- trial = str(reward.get("_trial_name") or reward.get("_trial_root_name") or "unknown trial")
- if trial in failed_trials:
+ raw_trial = str(reward.get("_trial_name") or reward.get("_trial_root_name") or "unknown trial")
+ if raw_trial in failed_trials:
continue
- failed_trials.add(trial)
+ failed_trials.add(raw_trial)
failures.append(
{
- "trial": trial,
+ "trial": _published_trial_label(raw_trial, alias_ordinal=len(failures) + 1),
"reason": _unscoreable_reward_reason(reward),
}
)
+
+ custom_names = {name for reward in scoreable for name in extract_custom_metrics(reward)}
+ if len(custom_names) > MAX_CUSTOM_METRICS:
+ for reward in scoreable:
+ _mark_reward_collection_failure(reward, UNSAFE_CUSTOM_METRIC_UNION_REASON)
+ raw_trial = str(reward.get("_trial_name") or reward.get("_trial_root_name") or "unknown trial")
+ if raw_trial in failed_trials:
+ continue
+ failed_trials.add(raw_trial)
+ failures.append(
+ {
+ "trial": _published_trial_label(raw_trial, alias_ordinal=len(failures) + 1),
+ "reason": UNSAFE_CUSTOM_METRIC_UNION_REASON,
+ }
+ )
+ scoreable = []
return scoreable, failures
def _unscoreable_reward_reason(reward: dict[str, Any]) -> str:
"""Return a bounded, redacted diagnostic for an unscoreable reward."""
- fallback = "Reward metrics are incomplete or non-finite; trial was not scored"
+ fallback = UNSCOREABLE_NUMERIC_REWARD_REASON
if str(reward.get("evaluation_status") or "").casefold() not in {"error", "failed"}:
return fallback
@@ -1845,9 +4015,21 @@ def _strip_attempt_suffix(value: str) -> str:
return re.sub(r"(?:[-_])attempt\d+$", "", value)
+def _reward_identity_is_publishable(reward: dict[str, Any]) -> bool:
+ """Validate the effective case identity before score aggregation."""
+ if reward.get("_trusted_case_identity_unresolved") is True:
+ return False
+ entry_id = reward.get("entry_id")
+ if entry_id not in (None, ""):
+ return _identity_text_is_publishable(entry_id)
+ trial_name = reward.get("_trial_name")
+ if not isinstance(trial_name, str) or not trial_name:
+ return False
+ return _identity_text_is_publishable(trial_name.split("__", 1)[0])
+
+
def _canonical_case_id(value: str, expected_case_ids: set[str] | None = None) -> str:
- value = str(value or "").strip()
- if not value:
+ if not _identity_text_is_publishable(value):
return ""
if expected_case_ids and value in expected_case_ids:
return value
@@ -1861,9 +4043,11 @@ def _canonical_case_id(value: str, expected_case_ids: set[str] | None = None) ->
def _entry_id(reward: dict[str, Any], expected_case_ids: set[str] | None = None) -> str:
- if reward.get("entry_id"):
- return _canonical_case_id(str(reward["entry_id"]), expected_case_ids)
- trial_name = str(reward.get("_trial_name") or "")
+ if reward.get("_trusted_case_identity_unresolved") is True:
+ return "unknown"
+ if isinstance(reward.get("entry_id"), str) and reward["entry_id"]:
+ return _canonical_case_id(reward["entry_id"], expected_case_ids)
+ trial_name = reward.get("_trial_name")
if trial_name:
return _canonical_case_id(trial_name.split("__", 1)[0], expected_case_ids)
return "unknown"
@@ -1872,19 +4056,41 @@ def _entry_id(reward: dict[str, Any], expected_case_ids: set[str] | None = None)
def _attempt_sort_key(reward: dict[str, Any]) -> tuple[int, int | str, str, str]:
"""Sort attempts by explicit attempt label, then Harbor start time."""
trial_name = str(reward.get("_trial_name") or "")
- match = re.search(r"attempt(\d+)", trial_name)
- if match:
- return (0, int(match.group(1)), str(reward.get("_started_at") or ""), trial_name)
+ attempt_ordinal = _attempt_ordinal(reward)
+ if attempt_ordinal is not None:
+ return (0, attempt_ordinal, str(reward.get("_started_at") or ""), trial_name)
started_at = str(reward.get("_started_at") or "")
return (1 if started_at else 2, started_at, "", trial_name)
def _attempt_ordinal(reward: dict[str, Any]) -> int | None:
- """Return an explicit Harbor attempt ordinal when the trial names carry one."""
+ """Return a carried ordinal or one unambiguous legacy suffix."""
+ if (attempt_ordinal := _positive_attempt_ordinal(reward.get("_attempt_ordinal"))) is not None:
+ return attempt_ordinal
+ if reward.get("_trusted_task_selector") is not None:
+ return None
+ entry_id = reward.get("entry_id")
+ if not isinstance(entry_id, str) or not entry_id:
+ return None
for key in ("_trial_root_name", "_trial_name"):
- match = re.search(r"attempt0*(\d+)", str(reward.get(key) or ""), flags=re.IGNORECASE)
- if match:
- return int(match.group(1))
+ trial_name = str(reward.get(key) or "")
+ if trial_name == entry_id:
+ continue
+ match = re.fullmatch(
+ r"(?P\d+)",
+ trial_name,
+ flags=re.IGNORECASE,
+ )
+ if not match:
+ continue
+ # Harbor 0.13 trial names embedded the logical task name immediately
+ # before ``_attemptNNN`` (sometimes behind a generated job prefix).
+ # Requiring that relationship preserves those artifacts without
+ # interpreting an authored selector/display name such as
+ # ``selector_attempt2`` as attempt two for an unrelated logical case.
+ base = match.group("base")
+ if base == entry_id or base.endswith((f"-{entry_id}", f"_{entry_id}")):
+ return _positive_attempt_ordinal(match.group("ordinal"))
return None
@@ -1903,13 +4109,18 @@ def _logical_attempt_rewards(rewards: list[dict[str, Any]]) -> list[dict[str, An
continue
first = rows[0]
standard_scores, metric_set, metrics = average_metrics(rows)
- custom_scores = average_custom_metrics(rows)
+ custom_scores = _average_multistep_custom_metrics(rows)
logical_reward: dict[str, Any] = {
"entry_id": first.get("entry_id"),
"_trial_name": root,
"_trial_root_name": root,
"_started_at": first.get("_started_at"),
+ "_logical_attempt_sentinel": _LOGICAL_ATTEMPT_SENTINEL,
}
+ if first.get("_trusted_task_selector") is not None:
+ logical_reward["_trusted_task_selector"] = first["_trusted_task_selector"]
+ if (attempt_ordinal := _attempt_ordinal(first)) is not None:
+ logical_reward["_attempt_ordinal"] = attempt_ordinal
if metric_set:
logical_reward["metric_set"] = metric_set
for metric in metrics:
@@ -1920,6 +4131,7 @@ def _logical_attempt_rewards(rewards: list[dict[str, Any]]) -> list[dict[str, An
logical_reward["custom_metrics"] = custom_scores
if (overall := _average_overall(rows)) is not None:
logical_reward["overall"] = overall
+ logical_reward["_logical_overall"] = overall
if any(row.get("_has_trajectory") for row in rows):
logical_reward["_has_trajectory"] = True
logical.append(logical_reward)
@@ -2106,8 +4318,8 @@ def _paired_pass_comparison(
without_skill: dict[str, Any],
) -> dict[str, Any]:
"""Compare pass@k outcomes for matching cases across both evaluation arms."""
- with_cases = with_skill.get("cases")
- without_cases = without_skill.get("cases")
+ with_cases = with_skill.get("_pairing_cases", with_skill.get("cases"))
+ without_cases = without_skill.get("_pairing_cases", without_skill.get("cases"))
if not isinstance(with_cases, dict) or not isinstance(without_cases, dict):
return {"pairing_status": "unavailable", "paired_cases": 0}
@@ -2203,6 +4415,11 @@ def _paired_pass_comparison(
return result
+def _public_pass_summary(summary: dict[str, Any]) -> dict[str, Any]:
+ """Remove collector-only pairing state from a pass summary."""
+ return {key: value for key, value in summary.items() if not key.startswith("_")}
+
+
def _pass_summary(
rewards: list[dict[str, Any]],
*,
@@ -2213,7 +4430,7 @@ def _pass_summary(
expected_case_ids: list[str] | None = None,
) -> dict[str, Any]:
"""Summarize pass@k using SkillEvaluator continuous reward scores."""
- expected_ids = list(dict.fromkeys(str(case_id) for case_id in (expected_case_ids or []) if str(case_id)))
+ expected_ids = _validated_expected_case_ids(expected_case_ids)
expected_id_set = set(expected_ids) if expected_ids else None
grouped: dict[str, list[dict[str, Any]]] = {}
for reward in _logical_attempt_rewards(rewards):
@@ -2222,18 +4439,22 @@ def _pass_summary(
grouped.setdefault(_entry_id(reward, expected_id_set), []).append(reward)
cases: dict[str, Any] = {}
+ pairing_cases: dict[str, dict[str, bool]] = {}
passed_cases = 0
attempts_used = 0
- extra_cases: list[str] = []
+ extra_case_ids: list[str] = []
case_order = expected_ids or sorted(grouped)
if expected_ids:
- extra_cases = sorted(entry_id for entry_id in grouped if entry_id not in expected_id_set)
- case_order = [*case_order, *extra_cases]
+ extra_case_ids = sorted(entry_id for entry_id in grouped if entry_id not in expected_id_set)
+ case_order = [*case_order, *extra_case_ids]
+
+ published_case_ids = set(case_order[:PUBLISHED_CASE_DETAILS_MAX])
+ published_attempt_details = 0
for entry_id in case_order:
attempts = grouped.get(entry_id, [])
- attempt_rows = []
+ attempt_rows: list[dict[str, Any]] = []
best_score: float | None = None
first_pass_attempt: int | None = None
for idx, reward in enumerate(sorted(attempts, key=_attempt_sort_key), start=1):
@@ -2245,14 +4466,20 @@ def _pass_summary(
if passed and first_pass_attempt is None:
first_pass_attempt = idx
best_score = score if best_score is None else max(best_score, score)
- attempt_rows.append(
- {
- "attempt": idx,
- "trial": reward.get("_trial_name", ""),
- "score": score,
- "passed": passed,
- }
- )
+ if (
+ entry_id in published_case_ids
+ and len(attempt_rows) < PUBLISHED_ATTEMPT_DETAILS_PER_CASE_MAX
+ and published_attempt_details < PUBLISHED_ATTEMPT_DETAILS_MAX
+ ):
+ attempt_rows.append(
+ {
+ "attempt": idx,
+ "trial": _published_trial_label(reward.get("_trial_name", "")),
+ "score": score,
+ "passed": passed,
+ }
+ )
+ published_attempt_details += 1
case_passed = first_pass_attempt is not None
is_expected_case = expected_id_set is None or entry_id in expected_id_set
@@ -2264,17 +4491,22 @@ def _pass_summary(
skipped = unscored if stop_on_pass and case_passed else 0
missing = 0 if skipped else unscored
- cases[entry_id] = {
- "passed": case_passed,
- "first_pass_attempt": first_pass_attempt,
- "attempts_used": len(attempts),
- "attempts_skipped": skipped,
- "attempts_missing": missing,
- "best_score": round(best_score, 4) if best_score is not None else None,
- "attempts": attempt_rows,
- }
- if not is_expected_case:
- cases[entry_id]["extra_case"] = True
+ pairing_cases[entry_id] = {"passed": case_passed, "extra_case": not is_expected_case}
+ if entry_id in published_case_ids:
+ cases[entry_id] = {
+ "passed": case_passed,
+ "first_pass_attempt": first_pass_attempt,
+ "attempts_used": len(attempts),
+ "attempts_skipped": skipped,
+ "attempts_missing": missing,
+ "best_score": round(best_score, 4) if best_score is not None else None,
+ "attempts": attempt_rows,
+ "attempt_details_total": len(attempts),
+ "attempt_details_shown": len(attempt_rows),
+ "attempt_details_truncated": len(attempt_rows) < len(attempts),
+ }
+ if not is_expected_case:
+ cases[entry_id]["extra_case"] = True
if expected_ids:
total_cases = len(expected_ids)
@@ -2298,8 +4530,15 @@ def _pass_summary(
"attempts_used": attempts_used,
"max_attempts_possible": total_cases * n_attempts,
"avg_attempts_used": round(attempts_used / total_cases, 4) if total_cases else 0.0,
- "extra_cases": extra_cases,
+ "extra_case_count": len(extra_case_ids),
+ "extra_cases": extra_case_ids[:PUBLISHED_CASE_ID_DIAGNOSTIC_SAMPLE_MAX],
+ "extra_cases_truncated": len(extra_case_ids) > PUBLISHED_CASE_ID_DIAGNOSTIC_SAMPLE_MAX,
+ "case_details_total": len(case_order),
+ "case_details_shown": len(cases),
+ "case_details_truncated": len(cases) < len(case_order),
+ "case_details_limit": PUBLISHED_CASE_DETAILS_MAX,
"cases": cases,
+ "_pairing_cases": pairing_cases,
}
@@ -2333,7 +4572,7 @@ def _compute_lift(
def _average_overall(rewards: list[dict[str, Any]]) -> float | None:
"""Average the pass/lift overall score across reward payloads."""
- values = [overall_score(reward) for reward in rewards]
+ values = [_overall_score(reward) for reward in rewards]
if not values or any(value is None for value in values):
return None
return round(sum(value for value in values if value is not None) / len(values), 4)
@@ -2391,22 +4630,152 @@ def _security_finding_signature(finding: dict[str, Any]) -> tuple[str, str]:
def _safe_trial_path_component(value: Any) -> str:
"""Return a portable single path component or an empty string."""
- component = str(value or "").strip()
+ raw_component = str(value or "")
+ component = raw_component.strip()
+ invalid_characters = '<>:"/\\|?*\x00'
+ stem = component.split(".", 1)[0].rstrip(" .").casefold()
+ reserved = {
+ "con",
+ "prn",
+ "aux",
+ "nul",
+ *(f"com{index}" for index in range(1, 10)),
+ *(f"lpt{index}" for index in range(1, 10)),
+ *(f"com{index}" for index in "¹²³"),
+ *(f"lpt{index}" for index in "¹²³"),
+ }
+ try:
+ utf8_bytes = len(component.encode("utf-8"))
+ utf16_units = len(component.encode("utf-16-le")) // 2
+ except UnicodeEncodeError:
+ return ""
if (
not component
+ or component != raw_component
or component in {".", ".."}
- or any(character in component for character in ("/", "\\", ":", "\x00"))
- or any(ord(character) < 32 or ord(character) == 127 for character in component)
+ or utf8_bytes > PORTABLE_TRIAL_COMPONENT_MAX_UNITS
+ or utf16_units > PORTABLE_TRIAL_COMPONENT_MAX_UNITS
+ or any(
+ ord(character) < 32 or ord(character) == 127 or character in invalid_characters for character in component
+ )
+ or component.endswith(".")
+ or stem in reserved
+ or contains_credential_value(component)
):
return ""
return component
+def _safe_trial_source_component(value: Any) -> str:
+ """Return an exact safe child name without applying output normalization."""
+ if not isinstance(value, str):
+ return ""
+ if (
+ not value
+ or value in {".", ".."}
+ or "/" in value
+ or "\x00" in value
+ or any(ord(character) < 32 or ord(character) == 127 for character in value)
+ ):
+ return ""
+ if os.name == "nt":
+ windows_path = PureWindowsPath(value)
+ if windows_path.drive or windows_path.is_absolute() or len(windows_path.parts) != 1:
+ return ""
+ return value
+
+
def _persisted_trial_name(reward: dict[str, Any]) -> tuple[str, str]:
"""Derive output and source names only from physical Harbor path components."""
- trial_root_name = _safe_trial_path_component(reward.get("_trial_root_name")) or "unknown"
+ trial_root_name = _safe_trial_source_component(reward.get("_trial_root_name")) or "unknown"
+ output_root_name = _safe_trial_path_component(trial_root_name) or "unknown"
step_name = _safe_trial_path_component(reward.get("_step_name"))
- return (f"{trial_root_name}__{step_name}" if step_name else trial_root_name), trial_root_name
+ return (f"{output_root_name}__{step_name}" if step_name else output_root_name), trial_root_name
+
+
+def _portable_trial_name_key(value: str) -> str:
+ """Normalize one output name for case-insensitive and Win32-compatible collision checks."""
+ return unicodedata.normalize("NFC", value.rstrip(" .").casefold())
+
+
+def _persisted_trial_names(
+ rewards: list[dict[str, Any]],
+ job_dir: Path | None,
+) -> list[tuple[str, str]]:
+ """Resolve distinct physical reward identities to distinct output directories."""
+ scored_names, _unscored_names = _persisted_trial_layout(rewards, job_dir)
+ return scored_names
+
+
+def _persisted_trial_layout(
+ rewards: list[dict[str, Any]],
+ job_dir: Path | None,
+) -> tuple[list[tuple[str, str]], dict[str, str]]:
+ """Allocate portable output names across scored and unscored physical trials."""
+ entries = [
+ (
+ (
+ str(reward.get("_trial_root_name") or ""),
+ str(reward.get("_step_name") or ""),
+ ),
+ *_persisted_trial_name(reward),
+ )
+ for reward in rewards
+ ]
+ scored_roots = {trial_root_name for _identity, _legacy_name, trial_root_name in entries}
+ unscored_sources: list[str] = []
+ if job_dir is not None:
+ with contextlib.suppress(OSError):
+ for child in sorted(job_dir.iterdir()):
+ kind, _unsafe_reason = _inspect_trial_directory(child)
+ if child.name not in scored_roots and (
+ kind == "link" or (kind == "directory" and _looks_like_trial_dir(child))
+ ):
+ unscored_sources.append(child.name)
+
+ preferred_names = [legacy_name for _identity, legacy_name, _trial_root_name in entries]
+ unsafe_preferred_indices = {
+ index
+ for index, (identity, legacy_name, _trial_root_name) in enumerate(entries)
+ if not _safe_trial_path_component(identity[0])
+ or (identity[1] and not _safe_trial_path_component(identity[1]))
+ or not _safe_trial_path_component(legacy_name)
+ }
+ for source_name in unscored_sources:
+ preferred = _safe_trial_path_component(source_name)
+ if not preferred:
+ unsafe_preferred_indices.add(len(preferred_names))
+ preferred_names.append(preferred or "unknown")
+ indices_by_name: dict[str, list[int]] = {}
+ for index, preferred_name in enumerate(preferred_names):
+ indices_by_name.setdefault(_portable_trial_name_key(preferred_name), []).append(index)
+
+ conflicting_indices = {
+ index for indices in indices_by_name.values() if len(indices) > 1 for index in indices
+ } | unsafe_preferred_indices
+ resolved_names = list(preferred_names)
+ used_name_keys = set(indices_by_name)
+ suffix = 1
+ for index in range(len(preferred_names)):
+ if index not in conflicting_indices:
+ continue
+ while True:
+ resolved_name = f"skillevaluator-trial-collision-{suffix:06d}"
+ suffix += 1
+ resolved_key = _portable_trial_name_key(resolved_name)
+ if resolved_key not in used_name_keys:
+ break
+ resolved_names[index] = resolved_name
+ used_name_keys.add(resolved_key)
+
+ scored_names = [
+ (resolved_names[index], trial_root_name)
+ for index, (_identity, _legacy_name, trial_root_name) in enumerate(entries)
+ ]
+ unscored_names = {
+ source_name: resolved_names[len(entries) + index] for index, source_name in enumerate(unscored_sources)
+ }
+ return scored_names, unscored_names
def _annotate_security_attribution(
@@ -2428,6 +4797,7 @@ def _annotate_security_attribution(
"unknown_no_baseline": 0,
"cases": {},
}
+ seen_cases: set[str] = set()
for reward in with_rewards:
entry_id = _entry_id(reward)
@@ -2445,6 +4815,7 @@ def _annotate_security_attribution(
case_status = "safe"
if with_findings:
case_status = "with_skill_unsafe"
+ attribution_plan: list[tuple[str, str]] = []
for finding in with_findings:
signature = _security_finding_signature(finding)
if not baseline_run:
@@ -2475,39 +4846,111 @@ def _annotate_security_attribution(
"not show target-skill use before the unsafe action."
)
summary["ambiguous_with_skill_only"] += 1
- finding["attribution"] = attribution
- finding["attribution_explanation"] = explanation
- security["attribution"] = with_findings[0].get("attribution")
- security["attribution_explanation"] = with_findings[0].get("attribution_explanation")
+ attribution_plan.append((attribution, explanation))
+
+ first_attribution, first_explanation = attribution_plan[0]
+ projected_details: dict[str, Any] | None = None
+ for projection in ("full", "labels", "aggregate"):
+ try:
+ candidate_reward = copy.deepcopy(reward)
+ candidate_details = candidate_reward.get("details")
+ candidate_security = (
+ candidate_details.get("security") if isinstance(candidate_details, dict) else None
+ )
+ if not isinstance(candidate_security, dict):
+ raise ValueError("security detail projection disappeared")
+ candidate_findings = _security_score_findings(candidate_reward)
+ if projection != "aggregate":
+ if len(candidate_findings) != len(attribution_plan):
+ raise ValueError("security finding projection changed cardinality")
+ for candidate_finding, (attribution, explanation) in zip(
+ candidate_findings,
+ attribution_plan,
+ strict=True,
+ ):
+ candidate_finding["attribution"] = attribution
+ if projection == "full":
+ candidate_finding["attribution_explanation"] = explanation
+ candidate_security["attribution"] = first_attribution
+ candidate_security["attribution_explanation"] = first_explanation
+ if projection == "labels":
+ candidate_security["attribution_completeness"] = (
+ "Per-finding attribution labels were retained; repeated explanations were omitted "
+ "to stay within artifact limits."
+ )
+ elif projection == "aggregate":
+ candidate_security["attribution_completeness"] = (
+ "Per-finding attribution was omitted because the expanded reward would exceed "
+ "artifact limits."
+ )
+ if not _reward_publication_projection_is_safe(candidate_reward):
+ raise ValueError("security attribution projection exceeds publication limits")
+ except (MemoryError, RecursionError, TypeError, ValueError):
+ continue
+ projected_details = candidate_details
+ break
+ if projected_details is not None:
+ reward["details"] = projected_details
elif baseline_findings:
case_status = "baseline_unsafe_with_skill_safe"
- security.setdefault("findings", []).append(
- {
- "type": "skill_reduced_unsafe_behavior",
- "severity": "info",
- "message": "Baseline had unsafe agent action, but with-skill run did not",
- "evidence": "; ".join(str(f.get("message", "")) for f in baseline_findings[:2]),
- "source": "baseline_comparison",
- "score_impact": False,
- "attribution": "skill_may_have_improved_safety",
- "attribution_explanation": (
- "The without-skill baseline showed unsafe behavior for this case, while the "
- "with-skill run did not."
- ),
- }
- )
- security["attribution"] = "skill_may_have_improved_safety"
- security["attribution_explanation"] = (
+ explanation = (
"The without-skill baseline showed unsafe behavior for this case, while the with-skill run did not."
)
+ for include_derived_finding in (True, False):
+ try:
+ candidate_reward = copy.deepcopy(reward)
+ candidate_details = candidate_reward.get("details")
+ candidate_security = (
+ candidate_details.get("security") if isinstance(candidate_details, dict) else None
+ )
+ if not isinstance(candidate_security, dict):
+ raise ValueError("security detail projection disappeared")
+ if include_derived_finding:
+ candidate_findings = candidate_security.get("findings")
+ if not isinstance(candidate_findings, list):
+ candidate_findings = []
+ candidate_security["findings"] = candidate_findings
+ candidate_findings.append(
+ {
+ "type": "skill_reduced_unsafe_behavior",
+ "severity": "info",
+ "message": "Baseline had unsafe agent action, but with-skill run did not",
+ "evidence": (
+ f"Without-skill baseline contained {len(baseline_findings)} "
+ "score-impacting security finding(s)."
+ ),
+ "source": "baseline_comparison",
+ "score_impact": False,
+ "attribution": "skill_may_have_improved_safety",
+ "attribution_explanation": explanation,
+ }
+ )
+ else:
+ candidate_security["attribution_completeness"] = (
+ "The derived comparison finding was omitted to stay within artifact limits."
+ )
+ candidate_security["attribution"] = "skill_may_have_improved_safety"
+ candidate_security["attribution_explanation"] = explanation
+ if not _reward_publication_projection_is_safe(candidate_reward):
+ raise ValueError("security improvement projection exceeds publication limits")
+ except (MemoryError, RecursionError, TypeError, ValueError):
+ continue
+ reward["details"] = candidate_details
+ break
summary["skill_may_have_improved_safety"] += 1
- summary["cases"][entry_id] = {
- "status": case_status,
- "with_skill_findings": len(with_findings),
- "baseline_findings": len(baseline_findings),
- }
+ seen_cases.add(entry_id)
+ if entry_id in summary["cases"] or len(summary["cases"]) < PUBLISHED_CASE_DETAILS_MAX:
+ summary["cases"][entry_id] = {
+ "status": case_status,
+ "with_skill_findings": len(with_findings),
+ "baseline_findings": len(baseline_findings),
+ }
+ summary["case_details_total"] = len(seen_cases)
+ summary["case_details_shown"] = len(summary["cases"])
+ summary["case_details_truncated"] = len(summary["cases"]) < len(seen_cases)
+ summary["case_details_limit"] = PUBLISHED_CASE_DETAILS_MAX
return summary
@@ -2525,37 +4968,106 @@ def _is_standard_skill_execution_reward(reward: dict[str, Any]) -> bool:
)
-def _trajectory_skill_invoked(trajectory: Any, skill_name: str) -> bool | None:
- """Derive target invocation from one readable ATIF trajectory."""
+def _trajectory_skill_invoked(trajectory: Any, skill_name: str, *, depth: int = 0) -> bool | None:
+ """Derive target invocation from executed root and referenced subagent paths."""
if not isinstance(trajectory, dict):
return None
+ if depth > _MAX_TRAJECTORY_REFERENCE_DEPTH:
+ return None
steps = trajectory.get("steps")
if not isinstance(steps, list) or not steps or any(not isinstance(step, dict) for step in steps):
return None
- agent_steps = [step for step in steps if step.get("source") == "agent"]
- if not agent_steps:
- return None
- for step in agent_steps:
- tool_calls = step.get("tool_calls")
- if not isinstance(tool_calls, list):
- return None
- for tool_call in tool_calls:
- if (
+ agent_steps = [
+ step for step in steps if step.get("source") == "agent" and step.get("is_copied_context") is not True
+ ]
+ results: list[bool | None] = []
+ if agent_steps:
+ for step in agent_steps:
+ tool_calls = step.get("tool_calls")
+ if tool_calls is None:
+ # ATIF 1.7 makes tool_calls optional. A normal agent message
+ # without calls contributes no routing evidence, but it must
+ # not hide a later, well-formed invocation.
+ continue
+ if not isinstance(tool_calls, list):
+ results.append(None)
+ break
+ if any(
not isinstance(tool_call, dict)
or not isinstance(tool_call.get("function_name"), str)
or not tool_call["function_name"].strip()
or not isinstance(tool_call.get("arguments"), dict)
+ for tool_call in tool_calls
):
- return None
- try:
- agent_trajectory = {**trajectory, "steps": agent_steps}
- tool_calls = extract_tool_calls_as_dicts(agent_trajectory)
- skill_tool_names = get_skill_tool_calls(agent_trajectory)
- negative_check = check_negative_case(tool_calls, skill_name, skill_tool_names=skill_tool_names)
- except (AttributeError, TypeError, ValueError):
+ results.append(None)
+ break
+ else:
+ try:
+ agent_trajectory = {**trajectory, "steps": agent_steps}
+ tool_calls = extract_tool_calls_as_dicts(agent_trajectory)
+ skill_tool_names = get_skill_tool_calls(agent_trajectory)
+ negative_check = check_negative_case(tool_calls, skill_name, skill_tool_names=skill_tool_names)
+ except (AttributeError, TypeError, ValueError):
+ results.append(None)
+ else:
+ passed = negative_check.get("passed")
+ results.append(not passed if isinstance(passed, bool) else None)
+ else:
+ results.append(None)
+
+ embedded = trajectory.get("subagent_trajectories")
+ if embedded is None:
+ embedded_by_id: dict[str, dict[str, Any]] = {}
+ elif not isinstance(embedded, list):
+ return True if True in results else None
+ else:
+ embedded_by_id = {}
+ for child in embedded:
+ if not isinstance(child, dict):
+ return True if True in results else None
+ trajectory_id = child.get("trajectory_id")
+ if not isinstance(trajectory_id, str) or not trajectory_id or trajectory_id in embedded_by_id:
+ return True if True in results else None
+ embedded_by_id[trajectory_id] = child
+
+ referenced_ids: list[str] = []
+ seen_ids: set[str] = set()
+ for step in steps:
+ if step.get("is_copied_context") is True:
+ continue
+ observation = step.get("observation")
+ if not isinstance(observation, dict):
+ continue
+ observation_results = observation.get("results")
+ if not isinstance(observation_results, list):
+ return True if True in results else None
+ for observation_result in observation_results:
+ if not isinstance(observation_result, dict):
+ return True if True in results else None
+ refs = observation_result.get("subagent_trajectory_ref")
+ if refs is None:
+ continue
+ if not isinstance(refs, list):
+ return True if True in results else None
+ for ref in refs:
+ if not isinstance(ref, dict):
+ return True if True in results else None
+ trajectory_id = ref.get("trajectory_id")
+ if not isinstance(trajectory_id, str) or trajectory_id not in embedded_by_id:
+ results.append(None)
+ continue
+ if trajectory_id not in seen_ids:
+ seen_ids.add(trajectory_id)
+ referenced_ids.append(trajectory_id)
+ results.extend(
+ _trajectory_skill_invoked(embedded_by_id[trajectory_id], skill_name, depth=depth + 1)
+ for trajectory_id in referenced_ids
+ )
+ if True in results:
+ return True
+ if None in results:
return None
- passed = negative_check.get("passed")
- return not passed if isinstance(passed, bool) else None
+ return False
def _authoritative_step_names(trial_root: Path) -> tuple[bool, list[str] | None]:
@@ -2589,47 +5101,71 @@ def _authoritative_step_names(trial_root: Path) -> tuple[bool, list[str] | None]
# Single-step Harbor trials serialize an explicit null. A physical
# steps layout still makes that shape ambiguous, so keep it fail-closed.
return (True, None) if steps_layout_present else (False, None)
- if not isinstance(step_results, list) or not step_results:
- return True, None
-
- names: list[str] = []
- for step in step_results:
- if not isinstance(step, dict):
- return True, None
- step_name = step.get("step_name")
- if not isinstance(step_name, str) or not step_name or step_name in names:
- return True, None
- names.append(step_name)
+ names = _valid_step_result_names(
+ step_results,
+ max_count=_MAX_TRAJECTORY_STEP_DIRECTORIES,
+ )
return True, names
-def _trusted_trial_skill_invoked(
+def _materialized_trial_trajectory(
trial_root: Path,
step_name: str | None,
- skill_name: str,
-) -> bool | None:
- """Derive trusted invocation without falling back across logical steps."""
- if step_name:
- safe_step_paths = {path.parent.parent.name: path for path in _ordered_step_trajectory_paths(trial_root)}
- trajectory_path = safe_step_paths.get(step_name)
- return _trajectory_skill_invoked(_read_json(trajectory_path), skill_name) if trajectory_path else None
-
+) -> tuple[dict[str, Any] | None, str | None]:
+ """Select and materialize the trajectory authorized by Harbor topology."""
is_multi_step, authoritative_names = _authoritative_step_names(trial_root)
+ root_path = trial_root / "agent" / "trajectory.json"
+ try:
+ root_path.lstat()
+ except FileNotFoundError:
+ root_present = False
+ except OSError:
+ return None, "trajectory_stat_failed"
+ else:
+ root_present = True
+
if is_multi_step:
if authoritative_names is None:
- return None
- safe_step_paths = {path.parent.parent.name: path for path in _ordered_step_trajectory_paths(trial_root)}
- saw_unknown = False
- for authoritative_name in authoritative_names:
- trajectory_path = safe_step_paths.get(authoritative_name)
- invoked = _trajectory_skill_invoked(_read_json(trajectory_path), skill_name) if trajectory_path else None
- if invoked is True:
- return True
- if invoked is None:
- saw_unknown = True
- return None if saw_unknown else False
+ return None, "invalid_or_incomplete_multi_step_topology"
+ if root_present:
+ return None, "contradictory_root_and_multi_step_trajectories"
+ paths = _ordered_step_trajectory_paths(trial_root)
+ discovered_names = [path.parent.parent.name for path in paths]
+ if discovered_names != authoritative_names:
+ return None, "incomplete_or_unexpected_multi_step_trajectories"
+ if step_name:
+ if step_name not in authoritative_names:
+ return None, "reward_step_not_in_authoritative_topology"
+ selected = paths[authoritative_names.index(step_name)]
+ try:
+ trajectory, _reference_key = _materialize_trajectory_file(selected.parent, selected.name)
+ except (OSError, SecurePathError, _TrajectoryMergeError, RecursionError):
+ return None, "invalid_step_trajectory"
+ return trajectory, None
+ merged = _merged_step_trajectory(trial_root)
+ if merged is None:
+ return None, "incomplete_or_invalid_multi_step_trajectory"
+ return merged, None
- return _trajectory_skill_invoked(_read_json(trial_root / "agent" / "trajectory.json"), skill_name)
+ if step_name:
+ return None, "unexpected_step_reward_for_single_step_trial"
+ if not root_present:
+ return None, "missing_single_step_trajectory"
+ try:
+ trajectory, _reference_key = _materialize_trajectory_file(root_path.parent, root_path.name)
+ except (OSError, SecurePathError, _TrajectoryMergeError, RecursionError):
+ return None, "invalid_single_step_trajectory"
+ return trajectory, None
+
+
+def _trusted_trial_skill_invoked(
+ trial_root: Path,
+ step_name: str | None,
+ skill_name: str,
+) -> bool | None:
+ """Derive trusted invocation from the topology-authorized materialized ATIF."""
+ trajectory, _reason = _materialized_trial_trajectory(trial_root, step_name)
+ return _trajectory_skill_invoked(trajectory, skill_name)
def _add_trusted_invocation_evidence(
@@ -2637,6 +5173,8 @@ def _add_trusted_invocation_evidence(
source_reward: dict[str, Any],
trial_root: Path | None,
skill_name: str,
+ *,
+ trajectory: dict[str, Any] | None | object = _TRAJECTORY_NOT_PROVIDED,
) -> None:
"""Replace verifier-authored routing evidence on standard rewards only."""
if not _is_standard_skill_execution_reward(source_reward):
@@ -2645,7 +5183,11 @@ def _add_trusted_invocation_evidence(
clean_reward.pop(key, None)
if trial_root is None:
return
- invoked = _trusted_trial_skill_invoked(trial_root, source_reward.get("_step_name"), skill_name)
+ invoked = (
+ _trajectory_skill_invoked(trajectory, skill_name)
+ if trajectory is not _TRAJECTORY_NOT_PROVIDED
+ else _trusted_trial_skill_invoked(trial_root, source_reward.get("_step_name"), skill_name)
+ )
if invoked is None:
return
if invoked is True:
@@ -2659,36 +5201,15 @@ def _add_trusted_invocation_evidence(
def _can_restore_custom_metric_name(value: str) -> bool:
"""Allow safe names plus narrow, explicitly metric-shaped secret terms."""
- if not is_sensitive_key(value):
- return True
- camel_split = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value)
- normalized = re.sub(r"[^a-zA-Z0-9]+", "_", camel_split).strip("_").lower()
- parts = tuple(part for part in normalized.split("_") if part)
- return (
- len(parts) == 2
- and parts[0] in {"auth", "secret", "token"}
- and parts[1]
- in {
- "accuracy",
- "compliance",
- "count",
- "coverage",
- "efficiency",
- "handling",
- "leakage",
- "precision",
- "quality",
- "rate",
- "ratio",
- "recall",
- "safety",
- "score",
- "usage",
- }
- )
+ return custom_metric_name_is_publishable(value)
-def _restore_custom_metric_scores(source_reward: dict[str, Any], safe_reward: dict[str, Any]) -> None:
+def _restore_custom_metric_scores(
+ source_reward: dict[str, Any],
+ safe_reward: dict[str, Any],
+ *,
+ max_str_len: int | None = None,
+) -> None:
"""Restore only finite numeric values recognized by the custom-metric schema."""
for field in ("custom_metrics", "metrics"):
source_metrics = source_reward.get(field)
@@ -2697,26 +5218,71 @@ def _restore_custom_metric_scores(source_reward: dict[str, Any], safe_reward: di
continue
for raw_name, raw_value in source_metrics.items():
name = str(raw_name)
- if not _can_restore_custom_metric_name(name):
+ if name not in safe_metrics or not _can_restore_custom_metric_name(name):
continue
score = extract_custom_metrics({field: {name: raw_value}}).get(name)
if score is None:
continue
if isinstance(raw_value, dict):
- safe_value = safe_metrics.get(name)
- if not isinstance(safe_value, dict):
- safe_value = {}
- safe_metrics[name] = safe_value
+ redacted_value = redact_sensitive_data(raw_value, max_str_len=max_str_len)
+ safe_value = redacted_value if isinstance(redacted_value, dict) else {}
+ safe_metrics[name] = safe_value
safe_value["score"] = score
else:
safe_metrics[name] = score
- # Legacy custom-only rewards allowed this report metric at the top level.
- token_efficiency = extract_custom_metrics({"token_efficiency": source_reward.get("token_efficiency")}).get(
- "token_efficiency"
- )
- if token_efficiency is not None:
- safe_reward["token_efficiency"] = token_efficiency
+ for name, score in extract_custom_metrics(source_reward).items():
+ if name not in source_reward or name not in safe_reward:
+ continue
+ raw_value = source_reward[name]
+ if isinstance(raw_value, dict):
+ redacted_value = redact_sensitive_data(raw_value, max_str_len=max_str_len)
+ safe_value = redacted_value if isinstance(redacted_value, dict) else {}
+ safe_value["score"] = score
+ safe_reward[name] = safe_value
+ else:
+ safe_reward[name] = score
+
+
+def _restore_custom_metric_details(
+ source_reward: dict[str, Any],
+ safe_reward: dict[str, Any],
+ *,
+ max_str_len: int | None = None,
+) -> None:
+ """Preserve safe metric-keyed evidence while redacting nested secrets."""
+ custom_names = set(extract_custom_metrics(source_reward))
+ source_custom_details = source_reward.get("custom_details")
+ if isinstance(source_custom_details, dict):
+ safe_custom_details = {
+ str(raw_name): redact_sensitive_data(detail, max_str_len=max_str_len)
+ for raw_name, detail in source_custom_details.items()
+ if str(raw_name) in custom_names
+ }
+ if safe_custom_details:
+ safe_reward["custom_details"] = safe_custom_details
+ else:
+ safe_reward.pop("custom_details", None)
+
+ source_details = source_reward.get("details")
+ safe_details = safe_reward.get("details")
+ if not isinstance(source_details, dict) or not isinstance(safe_details, dict):
+ return
+ for raw_name, detail in source_details.items():
+ name = str(raw_name)
+ if name in custom_names:
+ safe_details[name] = redact_sensitive_data(detail, max_str_len=max_str_len)
+
+
+def _strict_json_numbers(value: Any, *, max_nodes: int = COLLECTED_REWARD_JSON_MAX_NODES) -> Any:
+ """Replace non-finite floats before strict generated-artifact serialization."""
+ try:
+ normalized, _invalid = _normalized_reward_numbers(value, _max_nodes=max_nodes)
+ except (_RewardStructureLimitError, RecursionError, MemoryError):
+ if isinstance(value, dict):
+ return _structural_limit_reward(value)
+ return None
+ return normalized
def _save_trials(
@@ -2731,30 +5297,57 @@ def _save_trials(
agent_model_source: str | None = None,
) -> None:
"""Save per-trial reward.json and trajectory.json into the results directory."""
+ agent = _bounded_reward_metadata_text(agent) or "unknown"
+ agent_model = _bounded_reward_metadata_text(agent_model)
+ agent_model_source = _bounded_reward_metadata_text(agent_model_source)
trials_dir.mkdir(parents=True, exist_ok=True)
- for reward in rewards:
- trial_name, trial_root_name = _persisted_trial_name(reward)
+ persisted_names, unscored_names = _persisted_trial_layout(rewards, job_dir)
+ for reward, (trial_name, trial_root_name) in zip(rewards, persisted_names, strict=True):
trial_out = trials_dir / trial_name
trial_out.mkdir(parents=True, exist_ok=True)
trial_src = job_dir / trial_root_name if job_dir else None
- src_traj = _reward_trajectory_path(trial_src, reward.get("_step_name")) if trial_src else None
- merged_traj = (
- _merged_step_trajectory(trial_src)
- if trial_src and not reward.get("_step_name") and not (trial_src / "agent" / "trajectory.json").exists()
- else None
- )
- if merged_traj and "_trajectory_summary" not in reward:
- reward["_trajectory_summary"] = _summarize_trajectory(merged_traj)
- elif src_traj and src_traj.exists() and "_trajectory_summary" not in reward:
- reward["_trajectory_summary"] = _summarize_trajectory_file(src_traj)
+ materialized_traj: dict[str, Any] | None = None
+ trajectory_reason: str | None = None
+ if trial_src:
+ materialized_traj, trajectory_reason = _materialized_trial_trajectory(
+ trial_src,
+ reward.get("_step_name"),
+ )
+ safe_materialized_traj = _redacted_trajectory_data(materialized_traj) if materialized_traj else None
+ if materialized_traj is not None and safe_materialized_traj is None:
+ trajectory_reason = "trajectory_redaction_or_validation_failed"
+ if safe_materialized_traj is None:
+ reward.setdefault(
+ "_trajectory_summary",
+ {"readable": False, "reason": trajectory_reason or "trajectory_unavailable"},
+ )
+ elif "_trajectory_summary" not in reward:
+ reward["_trajectory_summary"] = _summarize_trajectory(materialized_traj)
clean_reward = {k: v for k, v in reward.items() if not k.startswith("_")}
- _add_trusted_invocation_evidence(clean_reward, reward, trial_src, skill_name)
+ if safe_materialized_traj is None and trajectory_reason not in {
+ None,
+ "missing_single_step_trajectory",
+ }:
+ warning = "Trajectory artifact omitted because it exceeded safety or validation limits."
+ warnings = clean_reward.get("warnings")
+ if not isinstance(warnings, list) or not all(isinstance(item, str) for item in warnings):
+ warnings = []
+ clean_reward["warnings"] = warnings
+ if warning not in warnings:
+ warnings.append(warning)
+ _add_trusted_invocation_evidence(
+ clean_reward,
+ reward,
+ trial_src,
+ skill_name,
+ trajectory=materialized_traj if safe_materialized_traj is not None else None,
+ )
# Persist the physical attempt identity so bounded report readers can keep
# fallback multi-step rows for diagnostics without weighting a logical
# Harbor trial once per step. This also populates the canonical report's
# existing trial_id field instead of inventing a second report schema.
- clean_reward["trial_id"] = trial_root_name
+ clean_reward["trial_id"] = _published_trial_label(trial_root_name)
if not clean_reward.get("entry_id"):
clean_reward["entry_id"] = _entry_id(reward)
clean_reward["agent"] = agent
@@ -2764,26 +5357,49 @@ def _save_trials(
clean_reward["model_source"] = agent_model_source
if "evaluation_errors" in clean_reward:
clean_reward["evaluation_errors"] = _safe_evaluation_errors(clean_reward["evaluation_errors"])
+ clean_reward = _sanitize_reward_metric_surfaces(clean_reward)
diagnostic_reward = (
str(clean_reward.get("evaluation_status") or "").casefold() in {"error", "failed"}
or overall_score(clean_reward) is None
)
+ max_str_len = REWARD_DIAGNOSTIC_STRING_MAX_CHARS if diagnostic_reward else None
safe_reward = redact_sensitive_data(
clean_reward,
- max_str_len=REWARD_DIAGNOSTIC_STRING_MAX_CHARS if diagnostic_reward else None,
+ max_str_len=max_str_len,
+ )
+ _restore_custom_metric_scores(clean_reward, safe_reward, max_str_len=max_str_len)
+ _restore_custom_metric_details(clean_reward, safe_reward, max_str_len=max_str_len)
+ safe_reward = _strict_json_numbers(safe_reward, max_nodes=REWARD_JSON_MAX_NODES)
+ if isinstance(safe_reward, dict):
+ safe_reward = _fail_closed_invalid_reward_numbers(
+ safe_reward,
+ max_nodes=REWARD_JSON_MAX_NODES,
+ max_bytes=GENERATED_JSON_MAX_BYTES,
+ )
+ (trial_out / "reward.json").write_text(
+ json.dumps(
+ safe_reward,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ),
+ encoding="utf-8",
)
- _restore_custom_metric_scores(clean_reward, safe_reward)
- (trial_out / "reward.json").write_text(json.dumps(safe_reward, indent=2), encoding="utf-8")
if trial_src:
- _copy_trial_artifacts(trial_src, trial_out)
- if merged_traj:
+ _copy_trial_artifacts(trial_src, trial_out, include_root_trajectory=False)
+ if safe_materialized_traj:
(trial_out / "trajectory.json").write_text(
- json.dumps(redact_sensitive_data(merged_traj), indent=2),
+ json.dumps(
+ safe_materialized_traj,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ),
encoding="utf-8",
)
- elif src_traj and src_traj.exists():
- _write_redacted_text_copy(src_traj, trial_out / "trajectory.json", source_root=trial_src)
+ elif trial_src:
+ _record_skipped_trajectory(trial_out, trajectory_reason or "trajectory_unavailable")
_save_unscored_trials(
rewards,
@@ -2793,6 +5409,7 @@ def _save_trials(
variant=variant,
agent_model=agent_model,
agent_model_source=agent_model_source,
+ persisted_names=unscored_names,
)
@@ -2857,27 +5474,46 @@ def _condition_execution_summary(
is not sufficient and raw reward-row count would over-count those tasks.
Early-stopped cases require attempts only through their first passing trial.
"""
- expected_ids = list(dict.fromkeys(str(case_id) for case_id in (expected_case_ids or []) if str(case_id)))
+ expected_ids = _validated_expected_case_ids(expected_case_ids)
expected_count = len(expected_ids) if expected_ids else int(expected_cases or 0)
if skipped:
return {
"execution_status": "skipped",
"execution_errors": [],
+ "execution_error_details_total": 0,
+ "execution_error_details_shown": 0,
+ "execution_error_details_truncated": False,
"expected_attempts": 0,
"scored_attempts": 0,
+ **_failure_list_metadata("runtime_failure_details", runtime_failures),
+ **_failure_list_metadata("reward_failure_details", reward_failures),
}
errors: list[str] = [job_failure] if job_failure else []
+ public_runtime_failures = _public_failure_list(runtime_failures)
+ public_reward_failures = _public_failure_list(reward_failures)
errors.extend(
- f"Agent runtime failed in {failure.get('trial', 'unknown trial')}: {failure.get('reason', 'unknown error')}"
- for failure in (runtime_failures or [])
+ "Agent runtime failed in "
+ f"{_published_trial_label(failure.get('trial', 'unknown trial'))}: "
+ f"{_safe_diagnostic_text(failure.get('reason', 'unknown error'), max_len=2048)}"
+ for failure in public_runtime_failures
)
errors.extend(
"Unscoreable reward in "
f"{_safe_diagnostic_text(failure.get('trial', 'unknown trial'), max_len=256)}: "
f"{_safe_diagnostic_text(failure.get('reason', 'unknown error'), max_len=2048)}"
- for failure in (reward_failures or [])
+ for failure in public_reward_failures
)
+ if len(runtime_failures or []) > len(public_runtime_failures):
+ errors.append(
+ "Agent runtime failure details were truncated "
+ f"(showing {len(public_runtime_failures)} of {len(runtime_failures or [])})"
+ )
+ if len(reward_failures or []) > len(public_reward_failures):
+ errors.append(
+ "Unscoreable reward details were truncated "
+ f"(showing {len(public_reward_failures)} of {len(reward_failures or [])})"
+ )
expected_set = set(expected_ids)
logical_passed: dict[str, bool] = {}
for reward in _logical_attempt_rewards(rewards):
@@ -2889,17 +5525,18 @@ def _condition_execution_summary(
roots: dict[str, dict[str, Any]] = {}
for reward in rewards:
root = str(reward.get("_trial_root_name") or "").strip()
+ published_root = _published_trial_label(root)
case_id = _entry_id(reward, expected_set or None)
step_name = str(reward.get("_step_name") or "").strip()
if not root:
errors.append("A scored reward is missing its Harbor trial root name")
continue
if not case_id or case_id == "unknown":
- errors.append(f"Scored trial {root!r} has no case identifier")
+ errors.append(f"Scored trial {published_root!r} has no case identifier")
continue
score = overall_score(reward)
if score is None:
- errors.append(f"Scored trial {root!r} has incomplete or non-finite reward metrics")
+ errors.append(f"Scored trial {published_root!r} has incomplete or non-finite reward metrics")
continue
existing = roots.get(root)
if existing is None:
@@ -2912,9 +5549,9 @@ def _condition_execution_summary(
}
continue
if existing["case_id"] != case_id:
- errors.append(f"Harbor trial {root!r} maps to multiple cases")
+ errors.append(f"Harbor trial {published_root!r} maps to multiple cases")
elif not step_name or step_name in existing["steps"]:
- errors.append(f"Harbor trial {root!r} has duplicate reward rows")
+ errors.append(f"Harbor trial {published_root!r} has duplicate reward rows")
else:
existing["steps"].add(step_name)
@@ -2966,7 +5603,7 @@ def _case_attempt_coverage(case_id: str, attempts: list[dict[str, Any]]) -> tupl
if expected_ids:
unexpected = sorted(case_id for case_id in by_case if case_id not in expected_set)
if unexpected:
- errors.append("Unexpected scored cases: " + ", ".join(unexpected))
+ errors.append(_sampled_case_id_diagnostic("Unexpected scored cases", unexpected))
else:
if expected_count and len(by_case) != expected_count:
errors.append(f"Scored case coverage is {len(by_case)}/{expected_count}")
@@ -2974,34 +5611,76 @@ def _case_attempt_coverage(case_id: str, attempts: list[dict[str, Any]]) -> tupl
expected_attempts += (expected_count - len(by_case)) * n_attempts
if missing:
- errors.append("Missing scored attempts for cases: " + ", ".join(sorted(missing)))
+ errors.append(_sampled_case_id_diagnostic("Missing scored attempts for cases", sorted(missing)))
if excess:
- errors.append("Excess scored attempts for cases: " + ", ".join(sorted(excess)))
+ errors.append(_sampled_case_id_diagnostic("Excess scored attempts for cases", sorted(excess)))
scored_attempts = len(roots)
if scored_attempts != expected_attempts:
errors.append(f"Scored attempt coverage is {scored_attempts}/{expected_attempts}")
- errors = list(dict.fromkeys(error for error in errors if error))
+ all_errors = list(
+ dict.fromkeys(
+ safe_error for error in errors if error if (safe_error := _safe_diagnostic_text(error, max_len=4096))
+ )
+ )
+ errors = all_errors[:PUBLISHED_EXECUTION_ERRORS_MAX]
return {
- "execution_status": "failed" if errors else "succeeded",
+ "execution_status": "failed" if all_errors else "succeeded",
"execution_errors": errors,
+ "execution_error_details_total": len(all_errors),
+ "execution_error_details_shown": len(errors),
+ "execution_error_details_truncated": len(errors) < len(all_errors),
"expected_attempts": expected_attempts,
"scored_attempts": scored_attempts,
+ **_failure_list_metadata("runtime_failure_details", runtime_failures),
+ **_failure_list_metadata("reward_failure_details", reward_failures),
}
def _aggregate_execution(summaries: list[dict[str, Any]]) -> dict[str, Any]:
+ """Aggregate execution status while retaining hidden child error counts.
+
+ Published error strings are deduplicated for display, but the total is an
+ occurrence count summed from child summaries. Hidden child strings cannot
+ be compared for uniqueness, so collapsing the total to the visible sample
+ would falsely erase declared diagnostics.
+ """
active = [summary for summary in summaries if summary.get("execution_status") != "skipped"]
- errors = [str(error) for summary in active for error in summary.get("execution_errors", []) if error]
+ all_errors = list(
+ dict.fromkeys(str(error) for summary in active for error in summary.get("execution_errors", []) if error)
+ )
+ errors = all_errors[:PUBLISHED_EXECUTION_ERRORS_MAX]
+ declared_total = 0
+ child_truncated = False
+ for summary in active:
+ raw_errors = summary.get("execution_errors")
+ visible_count = len([error for error in raw_errors if error]) if isinstance(raw_errors, list) else 0
+ raw_total = summary.get("execution_error_details_total")
+ child_total = (
+ raw_total
+ if isinstance(raw_total, int)
+ and not isinstance(raw_total, bool)
+ and 0 <= raw_total <= _MAX_JSON_SAFE_INTEGER
+ else visible_count
+ )
+ child_total = max(child_total, visible_count)
+ was_truncated = summary.get("execution_error_details_truncated") is True
+ if was_truncated and child_total <= visible_count:
+ child_total = min(_MAX_JSON_SAFE_INTEGER, visible_count + 1)
+ declared_total = min(_MAX_JSON_SAFE_INTEGER, declared_total + child_total)
+ child_truncated = child_truncated or was_truncated
if not active:
status = "skipped"
- elif errors or any(summary.get("execution_status") != "succeeded" for summary in active):
+ elif declared_total or any(summary.get("execution_status") != "succeeded" for summary in active):
status = "failed"
else:
status = "succeeded"
return {
"execution_status": status,
- "execution_errors": list(dict.fromkeys(errors)),
+ "execution_errors": errors,
+ "execution_error_details_total": declared_total,
+ "execution_error_details_shown": len(errors),
+ "execution_error_details_truncated": child_truncated or len(errors) < declared_total,
"expected_attempts": sum(int(summary.get("expected_attempts", 0) or 0) for summary in active),
"scored_attempts": sum(int(summary.get("scored_attempts", 0) or 0) for summary in active),
}
@@ -3019,6 +5698,7 @@ def collect_harbor_results(
stop_on_pass: bool = False,
expected_cases: int | None = None,
expected_case_ids: list[str] | None = None,
+ case_id_by_task_selector: dict[str, str] | None = None,
expected_trials: int | None = None,
expected_total_trials: int | None = None,
env_mode: str | None = None,
@@ -3033,6 +5713,11 @@ def collect_harbor_results(
raise ValueError("Conflicting expected trial counts were provided")
if expected_trials is None:
expected_trials = expected_total_trials
+ expected_case_ids = _validated_expected_case_ids(expected_case_ids)
+ case_id_by_task_selector = _validated_case_id_by_task_selector(
+ case_id_by_task_selector,
+ expected_case_ids,
+ )
all_results: dict[str, Any] = {
"agents": {},
@@ -3050,8 +5735,8 @@ def collect_harbor_results(
for agent in agents:
model_info = agent_models.get(agent, {}) if agent_models else {}
- agent_model = model_info.get("model")
- agent_model_source = model_info.get("source")
+ agent_model = _bounded_reward_metadata_text(model_info.get("model"))
+ agent_model_source = _bounded_reward_metadata_text(model_info.get("source"))
agent_dir = output_dir / agent
with_job_name = f"{skill_name}-{agent}-with"
@@ -3060,6 +5745,7 @@ def collect_harbor_results(
with_collected_rewards: list[dict[str, Any]] = []
with_rewards: list[dict[str, Any]] = []
with_logical_rewards: list[dict[str, Any]] = []
+ with_mixed_metric_contracts = False
with_scores: dict[str, float] = {}
with_custom_scores: dict[str, float] = {}
with_pass: dict[str, Any] = {}
@@ -3073,12 +5759,16 @@ def collect_harbor_results(
with_job_dir / "result.json",
expected_trials=expected_trials,
)
+ with_job_failure = _published_job_failure(with_job_failure)
with_runtime_failures = _extract_agent_runtime_failures(with_job_dir)
with_trial_failures = _extract_trial_failures(with_job_dir)
preserve_partial = _can_preserve_partial_rewards(with_job_dir, with_trial_failures)
- with_collected_rewards = _extract_rewards(with_job_dir) if with_job_ok or preserve_partial else []
+ with_collected_rewards = (
+ _extract_rewards(with_job_dir, case_id_by_task_selector) if with_job_ok or preserve_partial else []
+ )
with_rewards, invalid_score_failures = _partition_scoreable_rewards(with_collected_rewards)
with_logical_rewards = _logical_attempt_rewards(with_rewards)
+ with_mixed_metric_contracts = rewards_have_mixed_metric_contracts(with_rewards)
with_trial_failures.extend(invalid_score_failures)
with_scores, with_metric_set, with_metrics = average_metrics(with_logical_rewards)
all_results["metric_set"] = with_metric_set
@@ -3121,29 +5811,35 @@ def collect_harbor_results(
agent_model=agent_model,
agent_model_source=agent_model_source,
)
- (agent_dir / "with-skill" / "summary.json").write_text(
- json.dumps(
- {
- "agent": agent,
- "model": agent_model,
- "model_source": agent_model_source,
- "scores": with_scores,
- "custom_scores": with_custom_scores,
- "overall_score": with_overall_score,
- "metric_set": with_metric_set,
- "metrics": list(with_metrics),
- "dimensions": dimension_scores(with_scores),
- "num_trials": len(with_rewards),
- "pass_at_k": with_pass,
- **with_execution,
- "job_failure": with_job_failure,
- "trial_failures": with_trial_failures,
- },
- indent=2,
- ),
- encoding="utf-8",
+ _write_generated_root_json(
+ agent_dir / "with-skill" / "summary.json",
+ output_dir,
+ {
+ "agent": agent,
+ "model": agent_model,
+ "model_source": agent_model_source,
+ "scores": with_scores,
+ "custom_scores": with_custom_scores,
+ "overall_score": with_overall_score,
+ "metric_set": with_metric_set,
+ "metrics": list(with_metrics),
+ "dimensions": dimension_scores(with_scores),
+ "num_trials": len(with_logical_rewards),
+ "num_reward_rows": len(with_collected_rewards),
+ "mixed_metric_contracts": with_mixed_metric_contracts,
+ "pass_at_k": _public_pass_summary(with_pass),
+ **with_execution,
+ "job_failure": with_job_failure,
+ "trial_failures": _public_failure_list(with_trial_failures),
+ **_failure_list_metadata("trial_failure_details", with_trial_failures),
+ },
+ )
+ logger.debug(
+ "Agent %s with-skill: %d trials, scores=%s",
+ agent,
+ len(with_logical_rewards),
+ with_scores,
)
- logger.debug("Agent %s with-skill: %d trials, scores=%s", agent, len(with_rewards), with_scores)
else:
with_job_failure = f"No Harbor job found for {with_job_name}"
logger.warning("No Harbor job found for %s (with-skill)", with_job_name)
@@ -3152,28 +5848,29 @@ def collect_harbor_results(
(error.removeprefix(prefix) for error in (launch_errors or []) if error.startswith(prefix)),
f"Harbor job directory was not created: {with_job_name}",
)
+ with_job_failure = _published_job_failure(with_job_failure)
summary_dir = agent_dir / "with-skill"
summary_dir.mkdir(parents=True, exist_ok=True)
- (summary_dir / "summary.json").write_text(
- json.dumps(
- {
- "agent": agent,
- "model": agent_model,
- "model_source": agent_model_source,
- "scores": {},
- "custom_scores": {},
- "overall_score": None,
- "metric_set": DEFAULT_METRIC_SET,
- "metrics": list(DISPLAY_METRICS),
- "dimensions": {},
- "num_trials": 0,
- "pass_at_k": {},
- "job_failure": with_job_failure,
- "trial_failures": [],
- },
- indent=2,
- ),
- encoding="utf-8",
+ _write_generated_root_json(
+ summary_dir / "summary.json",
+ output_dir,
+ {
+ "agent": agent,
+ "model": agent_model,
+ "model_source": agent_model_source,
+ "scores": {},
+ "custom_scores": {},
+ "overall_score": None,
+ "metric_set": DEFAULT_METRIC_SET,
+ "metrics": list(DISPLAY_METRICS),
+ "dimensions": {},
+ "num_trials": 0,
+ "num_reward_rows": 0,
+ "mixed_metric_contracts": False,
+ "pass_at_k": {},
+ "job_failure": with_job_failure,
+ "trial_failures": [],
+ },
)
if not with_execution:
@@ -3190,31 +5887,32 @@ def collect_harbor_results(
if with_job_dir is None:
summary_dir = agent_dir / "with-skill"
summary_dir.mkdir(parents=True, exist_ok=True)
- (summary_dir / "summary.json").write_text(
- json.dumps(
- {
- "agent": agent,
- "model": agent_model,
- "model_source": agent_model_source,
- "scores": {},
- "custom_scores": {},
- "overall_score": None,
- "metrics": [],
- "dimensions": {},
- "num_trials": 0,
- "pass_at_k": {},
- **with_execution,
- "job_failure": with_job_failure,
- "trial_failures": [],
- },
- indent=2,
- ),
- encoding="utf-8",
+ _write_generated_root_json(
+ summary_dir / "summary.json",
+ output_dir,
+ {
+ "agent": agent,
+ "model": agent_model,
+ "model_source": agent_model_source,
+ "scores": {},
+ "custom_scores": {},
+ "overall_score": None,
+ "metrics": [],
+ "dimensions": {},
+ "num_trials": 0,
+ "num_reward_rows": 0,
+ "mixed_metric_contracts": False,
+ "pass_at_k": {},
+ **with_execution,
+ "job_failure": with_job_failure,
+ "trial_failures": [],
+ },
)
without_collected_rewards: list[dict[str, Any]] = []
without_rewards: list[dict[str, Any]] = []
without_logical_rewards: list[dict[str, Any]] = []
+ without_mixed_metric_contracts = False
without_scores: dict[str, float] = {}
without_custom_scores: dict[str, float] = {}
without_pass: dict[str, Any] = {}
@@ -3232,14 +5930,18 @@ def collect_harbor_results(
without_job_dir / "result.json",
expected_trials=expected_trials,
)
+ without_job_failure = _published_job_failure(without_job_failure)
without_runtime_failures = _extract_agent_runtime_failures(without_job_dir)
without_trial_failures = _extract_trial_failures(without_job_dir)
preserve_partial = _can_preserve_partial_rewards(without_job_dir, without_trial_failures)
without_collected_rewards = (
- _extract_rewards(without_job_dir) if without_job_ok or preserve_partial else []
+ _extract_rewards(without_job_dir, case_id_by_task_selector)
+ if without_job_ok or preserve_partial
+ else []
)
without_rewards, invalid_score_failures = _partition_scoreable_rewards(without_collected_rewards)
without_logical_rewards = _logical_attempt_rewards(without_rewards)
+ without_mixed_metric_contracts = rewards_have_mixed_metric_contracts(without_rewards)
without_trial_failures.extend(invalid_score_failures)
without_scores, without_metric_set, without_metrics = average_metrics(without_logical_rewards)
without_custom_scores = average_custom_metrics(without_logical_rewards)
@@ -3281,32 +5983,33 @@ def collect_harbor_results(
agent_model=agent_model,
agent_model_source=agent_model_source,
)
- (agent_dir / "without-skill" / "summary.json").write_text(
- json.dumps(
- {
- "agent": agent,
- "model": agent_model,
- "model_source": agent_model_source,
- "scores": without_scores,
- "custom_scores": without_custom_scores,
- "overall_score": without_overall_score,
- "metric_set": without_metric_set,
- "metrics": list(without_metrics),
- "dimensions": dimension_scores(without_scores),
- "num_trials": len(without_rewards),
- "pass_at_k": without_pass,
- **without_execution,
- "job_failure": without_job_failure,
- "trial_failures": without_trial_failures,
- },
- indent=2,
- ),
- encoding="utf-8",
+ _write_generated_root_json(
+ agent_dir / "without-skill" / "summary.json",
+ output_dir,
+ {
+ "agent": agent,
+ "model": agent_model,
+ "model_source": agent_model_source,
+ "scores": without_scores,
+ "custom_scores": without_custom_scores,
+ "overall_score": without_overall_score,
+ "metric_set": without_metric_set,
+ "metrics": list(without_metrics),
+ "dimensions": dimension_scores(without_scores),
+ "num_trials": len(without_logical_rewards),
+ "num_reward_rows": len(without_collected_rewards),
+ "mixed_metric_contracts": without_mixed_metric_contracts,
+ "pass_at_k": _public_pass_summary(without_pass),
+ **without_execution,
+ "job_failure": without_job_failure,
+ "trial_failures": _public_failure_list(without_trial_failures),
+ **_failure_list_metadata("trial_failure_details", without_trial_failures),
+ },
)
logger.debug(
"Agent %s without-skill: %d trials, scores=%s",
agent,
- len(without_rewards),
+ len(without_logical_rewards),
without_scores,
)
else:
@@ -3317,28 +6020,29 @@ def collect_harbor_results(
(error.removeprefix(prefix) for error in (launch_errors or []) if error.startswith(prefix)),
f"Harbor job directory was not created: {without_job_name}",
)
+ without_job_failure = _published_job_failure(without_job_failure)
summary_dir = agent_dir / "without-skill"
summary_dir.mkdir(parents=True, exist_ok=True)
- (summary_dir / "summary.json").write_text(
- json.dumps(
- {
- "agent": agent,
- "model": agent_model,
- "model_source": agent_model_source,
- "scores": {},
- "custom_scores": {},
- "overall_score": None,
- "metric_set": DEFAULT_METRIC_SET,
- "metrics": list(DISPLAY_METRICS),
- "dimensions": {},
- "num_trials": 0,
- "pass_at_k": {},
- "job_failure": without_job_failure,
- "trial_failures": [],
- },
- indent=2,
- ),
- encoding="utf-8",
+ _write_generated_root_json(
+ summary_dir / "summary.json",
+ output_dir,
+ {
+ "agent": agent,
+ "model": agent_model,
+ "model_source": agent_model_source,
+ "scores": {},
+ "custom_scores": {},
+ "overall_score": None,
+ "metric_set": DEFAULT_METRIC_SET,
+ "metrics": list(DISPLAY_METRICS),
+ "dimensions": {},
+ "num_trials": 0,
+ "num_reward_rows": 0,
+ "mixed_metric_contracts": False,
+ "pass_at_k": {},
+ "job_failure": without_job_failure,
+ "trial_failures": [],
+ },
)
if not without_execution:
@@ -3356,26 +6060,26 @@ def collect_harbor_results(
if not skip_baseline and without_job_dir is None:
summary_dir = agent_dir / "without-skill"
summary_dir.mkdir(parents=True, exist_ok=True)
- (summary_dir / "summary.json").write_text(
- json.dumps(
- {
- "agent": agent,
- "model": agent_model,
- "model_source": agent_model_source,
- "scores": {},
- "custom_scores": {},
- "overall_score": None,
- "metrics": [],
- "dimensions": {},
- "num_trials": 0,
- "pass_at_k": {},
- **without_execution,
- "job_failure": without_job_failure,
- "trial_failures": [],
- },
- indent=2,
- ),
- encoding="utf-8",
+ _write_generated_root_json(
+ summary_dir / "summary.json",
+ output_dir,
+ {
+ "agent": agent,
+ "model": agent_model,
+ "model_source": agent_model_source,
+ "scores": {},
+ "custom_scores": {},
+ "overall_score": None,
+ "metrics": [],
+ "dimensions": {},
+ "num_trials": 0,
+ "num_reward_rows": 0,
+ "mixed_metric_contracts": False,
+ "pass_at_k": {},
+ **without_execution,
+ "job_failure": without_job_failure,
+ "trial_failures": [],
+ },
)
lift: dict[str, Any] = {}
@@ -3385,7 +6089,7 @@ def collect_harbor_results(
)
if paired_execution_succeeded and with_scores and without_scores:
lift = _compute_lift(with_scores, without_scores)
- (agent_dir / "lift.json").write_text(json.dumps(lift, indent=2), encoding="utf-8")
+ _write_generated_root_json(agent_dir / "lift.json", output_dir, lift)
custom_lift: dict[str, Any] = {}
if (
@@ -3402,7 +6106,7 @@ def collect_harbor_results(
include_overall=not with_scores and not without_scores,
)
if custom_lift:
- (agent_dir / "custom_lift.json").write_text(json.dumps(custom_lift, indent=2), encoding="utf-8")
+ _write_generated_root_json(agent_dir / "custom_lift.json", output_dir, custom_lift)
pass_lift: dict[str, Any] = {}
if paired_execution_succeeded and with_pass and without_pass:
@@ -3414,7 +6118,7 @@ def collect_harbor_results(
"passed_cases_delta": int(with_pass.get("passed_cases", 0)) - int(without_pass.get("passed_cases", 0)),
"paired_comparison": _paired_pass_comparison(with_pass, without_pass),
}
- (agent_dir / "pass_at_k_lift.json").write_text(json.dumps(pass_lift, indent=2), encoding="utf-8")
+ _write_generated_root_json(agent_dir / "pass_at_k_lift.json", output_dir, pass_lift)
security_attribution: dict[str, Any] = {}
attribution_execution_succeeded = with_execution.get("execution_status") == "succeeded" and (
@@ -3426,8 +6130,10 @@ def collect_harbor_results(
without_rewards,
baseline_run=not skip_baseline,
)
- (agent_dir / "security_attribution.json").write_text(
- json.dumps(security_attribution, indent=2), encoding="utf-8"
+ _write_generated_root_json(
+ agent_dir / "security_attribution.json",
+ output_dir,
+ security_attribution,
)
if with_job_dir:
_save_trials(
@@ -3458,18 +6164,24 @@ def collect_harbor_results(
"lift": lift,
"custom_lift": custom_lift,
"pass_at_k": {
- "with_skill": with_pass,
- "without_skill": without_pass,
+ "with_skill": _public_pass_summary(with_pass),
+ "without_skill": _public_pass_summary(without_pass),
"lift": pass_lift,
},
"security_attribution": security_attribution,
"agent_runtime_failures": {
- "with_skill": with_runtime_failures,
- "without_skill": without_runtime_failures,
+ "with_skill": _public_failure_list(with_runtime_failures),
+ "without_skill": _public_failure_list(without_runtime_failures),
},
"trial_failures": {
- "with_skill": with_trial_failures,
- "without_skill": without_trial_failures,
+ "with_skill": _public_failure_list(with_trial_failures),
+ "without_skill": _public_failure_list(without_trial_failures),
+ },
+ "failure_detail_metadata": {
+ "with_skill_runtime": _failure_list_metadata("details", with_runtime_failures),
+ "without_skill_runtime": _failure_list_metadata("details", without_runtime_failures),
+ "with_skill_trials": _failure_list_metadata("details", with_trial_failures),
+ "without_skill_trials": _failure_list_metadata("details", without_trial_failures),
},
"job_failures": {
"with_skill": with_job_failure,
@@ -3480,8 +6192,8 @@ def collect_harbor_results(
"without_skill": without_execution,
},
**agent_execution,
- "num_trials_with": len(with_rewards),
- "num_trials_without": len(without_rewards) if not skip_baseline else 0,
+ "num_trials_with": len(with_logical_rewards),
+ "num_trials_without": len(without_logical_rewards) if not skip_baseline else 0,
"output_dir": str(agent_dir.resolve()),
}
diff --git a/src/skillevaluator/tier3/harbor/local_agents.py b/src/skillevaluator/tier3/harbor/local_agents.py
index 605ecc34..e0a908e8 100644
--- a/src/skillevaluator/tier3/harbor/local_agents.py
+++ b/src/skillevaluator/tier3/harbor/local_agents.py
@@ -80,22 +80,29 @@ async def _close_running_bridge(
await _await_task_uninterruptibly(close_task, preserve_cancellation=preserve_cancellation)
-def _codex_config_toml(base_url: str) -> str:
- """Generate Codex config for an OpenAI-compatible Responses API endpoint.
-
- The base URL comes from the caller's OPENAI_BASE_URL so an OSS user points
- Codex at their own OpenAI-compatible Responses provider; no endpoint is
- hardcoded.
- """
- return (
- "# Auto-generated by SkillEvaluator for OpenAI-compatible Responses routing.\n"
- 'model_provider = "openai_compatible"\n\n'
- "[model_providers.openai_compatible]\n"
- 'name = "OpenAI-compatible provider"\n'
- f'base_url = "{base_url}"\n'
- 'env_key = "OPENAI_API_KEY"\n'
- 'wire_api = "responses"\n'
+def _merge_codex_openai_compatible_config(config: dict[str, Any], base_url: str) -> dict[str, Any]:
+ """Apply SkillEvaluator's Responses routing without discarding user config."""
+ config["model_provider"] = "openai_compatible"
+
+ model_providers = config.get("model_providers")
+ if not isinstance(model_providers, dict):
+ model_providers = {}
+ config["model_providers"] = model_providers
+
+ provider = model_providers.get("openai_compatible")
+ if not isinstance(provider, dict):
+ provider = {}
+ model_providers["openai_compatible"] = provider
+
+ provider.update(
+ {
+ "name": "OpenAI-compatible provider",
+ "base_url": base_url,
+ "env_key": "OPENAI_API_KEY",
+ "wire_api": "responses",
+ }
)
+ return config
def _rewrite_launcher_segment(command: str, rewrite: Callable[[str], str]) -> str:
@@ -155,6 +162,12 @@ async def install(self, environment: BaseEnvironment) -> None:
def get_version_command(self) -> str | None:
return "codex --version"
+ def _build_effective_config(self, openai_base_url: str | None = None) -> dict[str, Any]:
+ """Extend Harbor's final user/MCP config with SkillEvaluator routing."""
+ config = super()._build_effective_config(openai_base_url)
+ base_url = openai_base_url or os.environ.get("OPENAI_BASE_URL") or _DEFAULT_OPENAI_BASE_URL
+ return _merge_codex_openai_compatible_config(config, base_url)
+
def _preserve_gateway_model_name(self, command: str, env: dict[str, str] | None) -> str:
"""Undo Harbor's default Codex model truncation when routing through an OpenAI-compatible gateway."""
model_name = str(self.model_name or "")
@@ -194,23 +207,6 @@ async def exec_as_agent(
timeout_sec=timeout_sec,
)
- async def run(self, instruction, environment: BaseEnvironment, context) -> None: # type: ignore[no-untyped-def]
- if getattr(self, "_nvidia_build_bridge_client_env", None) is None:
- config_path = self._REMOTE_CODEX_HOME / "config.toml"
- base_url = os.environ.get("OPENAI_BASE_URL") or _DEFAULT_OPENAI_BASE_URL
- config_text = _codex_config_toml(base_url).replace("'", "'\"'\"'")
- await self.exec_as_agent(
- environment,
- command=(
- f"mkdir -p {self._REMOTE_CODEX_HOME.as_posix()} "
- f"{self._REMOTE_CODEX_SECRETS_DIR.as_posix()} && "
- f"cat > {config_path.as_posix()} <<'EOF'\n"
- f"{config_text}"
- "EOF\n"
- ),
- )
- await super().run(instruction=instruction, environment=environment, context=context)
-
class SkillEvaluatorLocalOpenCode(OpenCode):
"""OpenCode wrapper that skips nvm/npm bootstrap in local mode."""
@@ -283,6 +279,7 @@ async def run(self, instruction, environment: BaseEnvironment, context) -> None:
return
instruction = self.render_instruction(instruction)
+ self._instruction = instruction
env = {key: os.environ[key] for key in ("OPENAI_API_KEY", "OPENAI_BASE_URL") if key in os.environ}
env = self._confined_project_env(env)
@@ -310,6 +307,9 @@ async def run(self, instruction, environment: BaseEnvironment, context) -> None:
env=env,
)
+ if messages := self._error_messages():
+ raise NonZeroAgentExitCodeError("OpenCode emitted error event(s): " + "; ".join(messages[:3]))
+
async def exec_as_agent(
self,
environment: BaseEnvironment,
@@ -339,6 +339,7 @@ async def exec_as_agent(
class _NvidiaBuildBridgeAgent:
"""Run a Docker-only loopback bridge around Harbor's stock vendor agent."""
+ _BRIDGE_CLIENT_ENV_UNSET = ("NVIDIA_API_KEY",)
_BRIDGE_SCRIPT = PurePosixPath(EnvironmentPaths.agent_dir / "nvidia-build-bridge.py")
_BRIDGE_PID = PurePosixPath(EnvironmentPaths.agent_dir / "nvidia-build-bridge.pid")
_BRIDGE_READY = PurePosixPath(EnvironmentPaths.agent_dir / "nvidia-build-bridge.ready")
@@ -468,9 +469,9 @@ async def _start_bridge(self, environment: BaseEnvironment) -> None:
# The selected secure Docker environment streams ``env=`` values
# over Compose stdin. Materialize both secrets only inside the
# not-yet-exposed task container, then discard the transient names.
- # Do not route this through BaseInstalledAgent._exec: Harbor 0.13.2
- # attaches that helper's raw ``env`` mapping to DEBUG LogRecords,
- # which structured/custom handlers may serialize.
+ # Route this through the selected environment's sensitive-exec
+ # transport so neither bridge capability enters process arguments
+ # or the agent's longer-lived execution environment.
handoff_result = await sensitive_exec(
command=(
"set -eu; umask 077; "
@@ -588,27 +589,20 @@ async def exec_as_agent(
client_env = getattr(self, "_nvidia_build_bridge_client_env", None)
if client_env is None:
return await super().exec_as_agent(environment, command=command, env=env, cwd=cwd, timeout_sec=timeout_sec)
- sensitive_exec = self._sensitive_exec(environment)
routed_env = dict(env or {})
- routed_env.update(getattr(self, "_extra_env", {}) or {})
routed_env.update(client_env)
- routed_env.pop("NVIDIA_API_KEY", None)
+ for name in self._BRIDGE_CLIENT_ENV_UNSET:
+ routed_env.pop(name, None)
routed_command = self._rewrite_bridge_client_command(command)
- # Harbor 0.13.2 attaches BaseInstalledAgent._exec's merged env to DEBUG
- # LogRecords, which structured/custom handlers may serialize.
- # Call the selected secure environment directly so its stdin transport
- # can protect the per-trial bridge capability as well.
- result = await sensitive_exec(
- command=(f"env -u NVIDIA_API_KEY bash -o pipefail -c {shlex.quote(routed_command)}"),
- env=routed_env,
- cwd=cwd,
- timeout_sec=timeout_sec,
- )
- if result.return_code != 0:
- raise NonZeroAgentExitCodeError(
- f"NVIDIA Build bridge client command failed with exit code {result.return_code}"
+ unset_args = " ".join(f"-u {shlex.quote(name)}" for name in self._BRIDGE_CLIENT_ENV_UNSET)
+ with environment.scoped_exec_env(client_env):
+ return await super().exec_as_agent(
+ environment,
+ command=f"env {unset_args} bash -o pipefail -c {shlex.quote(routed_command)}",
+ env=routed_env,
+ cwd=cwd,
+ timeout_sec=timeout_sec,
)
- return result
async def run(self, instruction, environment: BaseEnvironment, context) -> None: # type: ignore[no-untyped-def]
self._nvidia_build_bridge_started = False
@@ -639,33 +633,30 @@ async def run(self, instruction, environment: BaseEnvironment, context) -> None:
class SkillEvaluatorNvidiaBuildCodex(_NvidiaBuildBridgeAgent, Codex):
"""Stock Codex CLI routed through the in-trial NVIDIA Build bridge."""
+ _BRIDGE_CLIENT_ENV_UNSET = ("NVIDIA_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE")
+
def _bridge_client_environment(self) -> dict[str, str]:
return {
"OPENAI_API_KEY": self._bridge_client_token(),
}
- def _get_env(self, key: str) -> str | None:
+ def _get_env(self, key: str, *alternatives: str) -> str | None:
"""Expose the sentinel key while suppressing Codex's legacy base-url config."""
client_env = getattr(self, "_nvidia_build_bridge_client_env", None)
if client_env is not None:
if key == "OPENAI_BASE_URL":
return None
- if key in client_env:
- return client_env[key]
- return super()._get_env(key)
-
- async def _prepare_bridge_client(self, environment: BaseEnvironment) -> None:
- config_path = self._REMOTE_CODEX_HOME / "config.toml"
- config_text = _codex_config_toml(self._bridge_api_url())
- await self.exec_as_agent(
- environment,
- command=(
- f"mkdir -p {shlex.quote(self._REMOTE_CODEX_HOME.as_posix())} && "
- f"cat > {shlex.quote(config_path.as_posix())} <<'EOF'\n"
- f"{config_text}"
- "EOF\n"
- ),
- )
+ for name in (key, *alternatives):
+ if name in client_env:
+ return client_env[name]
+ return super()._get_env(key, *alternatives)
+
+ def _build_effective_config(self, openai_base_url: str | None = None) -> dict[str, Any]:
+ """Make the live bridge the final Codex route after user/MCP merging."""
+ _ = openai_base_url
+ bridge_api_url = self._bridge_api_url()
+ config = super()._build_effective_config(bridge_api_url)
+ return _merge_codex_openai_compatible_config(config, bridge_api_url)
def _rewrite_bridge_client_command(self, command: str) -> str:
model_name = str(self.model_name or "")
diff --git a/src/skillevaluator/tier3/harbor/local_environment.py b/src/skillevaluator/tier3/harbor/local_environment.py
index 9be41907..8e1566aa 100644
--- a/src/skillevaluator/tier3/harbor/local_environment.py
+++ b/src/skillevaluator/tier3/harbor/local_environment.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import asyncio
+import codecs
import contextlib
import json
import os
@@ -16,9 +17,9 @@
import subprocess
import sys
from pathlib import Path
-from typing import Any
+from typing import Any, NoReturn
-from harbor.environments.base import BaseEnvironment, ExecResult
+from harbor.environments.base import BaseEnvironment, ExecResult, OutputCallback, OutputStream
from harbor.environments.capabilities import EnvironmentCapabilities
from harbor.models.environment_type import EnvironmentType
@@ -31,8 +32,9 @@
runtime_command_roots,
validate_runtime_root,
)
-from skillevaluator.tier3.harbor.secret_redaction import redact_secrets_in_log_line
+from skillevaluator.tier3.harbor.progress import secret_values_from_environment
from skillevaluator.tier3.harbor.secure_copy import copytree_secure
+from skillevaluator.tier3.harbor.stream_redaction import CommandOutputByteBudget, StreamingLogRedactor
from skillevaluator.tier3.output_provenance import output_provenance_key_path
_SAFE_HOST_ENV = frozenset(
@@ -90,7 +92,28 @@
"ANTHROPIC_BASE_URL",
}
)
-_SECRET_ENV_NAME_RE = re.compile(r"(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|AUTH)", re.IGNORECASE)
+_SENSITIVE_ENV_NAME_RE = re.compile(
+ r"(?:^|_)(?:API_?KEY|ACCESS_?KEY|PRIVATE_?KEY|KEY|PAT|TOKEN|SECRET|PASS(?:WORD)?|"
+ r"CREDENTIALS?|AUTH(?:ORIZATION)?|BEARER|COOKIE|SESSION|CERT(?:IFICATE)?|DSN|"
+ r"CONNECTION(?:_STRING)?|(?:PRE)?SIGNED_?URL|SAS_?URL|CREDENTIAL_?URL|DATABASE_?URL)(?:_|$)",
+ re.IGNORECASE,
+)
+_LEGACY_SECRET_ENV_NAME_RE = re.compile(
+ r"(?:TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|AUTH)",
+ re.IGNORECASE,
+)
+# Compact credential names remain high-signal even without underscore token
+# boundaries. Deliberately omit a bare KEY suffix so ordinary names such as
+# MONKEY and KEYBOARD do not cause short public values to be rewritten.
+_COMPACT_SENSITIVE_ENV_NAME_RE = re.compile(
+ r"(?:^|_)[A-Z0-9]*(?:API_?KEY|ACCESS_?KEY|PRIVATE_?KEY|TOKEN|SECRET|PASSWORD|"
+ r"CREDENTIALS?|AUTHENTICATION|AUTHORIZATION|BEARER)(?:_|$)",
+ re.IGNORECASE,
+)
+# Before Harbor 0.22, local output used the broad legacy name matcher above.
+# Retain protection for credential-sized values while avoiding corruption from
+# common short variables such as MONKEY=banana and KEYBOARD=clacky.
+_MIN_LEGACY_SECRET_VALUE_LENGTH = 8
_SHELL_WRITE_REDIRECT_RE = re.compile(r"(?:^|\s)(?:\d?>{1,2}|&>)\s*([^\s;&|]+)")
_SHELL_WRITE_COMMAND_RE = re.compile(r"(?:^|[;&|]\s*)(?:tee|touch|mkdir|cp|mv)\b(?P[^;&|]*)")
_BACKGROUND_AMPERSAND_RE = re.compile(r"(?])&(?![&>])")
@@ -100,6 +123,8 @@
_REAP_TERM_SECONDS = 1.0
_REAP_KILL_SECONDS = 1.0
_REAP_CANCEL_SECONDS = 0.1
+_CREATION_CANCEL_SECONDS = 0.1
+_STOP_CLEANUP_SECONDS = _REAP_TERM_SECONDS + _REAP_KILL_SECONDS + _REAP_CANCEL_SECONDS + 0.2
_PATH_START_BOUNDARY_RE = r"(?])"
_HOST_HOME_PREFIX_RE = r"(?:~|\$HOME|\$\{HOME\}|/Users/[^\s/;'\"&|<>]+|/home/[^\s/;'\"&|<>]+|/root)"
@@ -126,6 +151,97 @@
)
+def _credential_uri_environment_values(environment: dict[str, str]) -> set[str]:
+ """Extract only credential URI/proxy values beyond local's name policy."""
+ candidates = {
+ name: value
+ for name, value in environment.items()
+ if value and (name.upper().endswith("_PROXY") or "://" in value)
+ }
+ return secret_values_from_environment(candidates)
+
+
+class _StreamCallbackOutput:
+ """Own per-stream redaction state until the exec outcome is known."""
+
+ def __init__(
+ self,
+ callback: OutputCallback,
+ callback_error: asyncio.Future[BaseException],
+ secret_values: set[str],
+ ) -> None:
+ self._callback = callback
+ self._callback_error = callback_error
+ self._redactors = {
+ "stdout": StreamingLogRedactor(secret_values),
+ "stderr": StreamingLogRedactor(secret_values),
+ }
+ self._raw_output: dict[OutputStream, bytearray] = {"stdout": bytearray(), "stderr": bytearray()}
+ self._delivery_cancelled = False
+ self._active_deliveries: set[asyncio.Task[None]] = set()
+
+ def append_raw(self, chunk: bytes, stream: OutputStream) -> None:
+ self._raw_output[stream].extend(chunk)
+
+ def raw_output(self, stream: OutputStream) -> bytes:
+ return bytes(self._raw_output[stream])
+
+ def abandon_delivery(self) -> None:
+ """Prevent timeout cleanup from re-entering a stuck callback."""
+ self._delivery_cancelled = True
+
+ async def _emit(self, text: str, stream: OutputStream) -> None:
+ if not text or self._callback_error.done() or self._delivery_cancelled:
+ return
+
+ async def invoke_callback() -> None:
+ try:
+ await self._callback(text, stream)
+ except asyncio.CancelledError as exc:
+ current = asyncio.current_task()
+ if current is not None and current.cancelling():
+ # Cleanup is cancelling this delivery task. A deliberate
+ # CancelledError raised by callback code has a zero
+ # cancellation count and remains the callback failure.
+ raise
+ if not self._callback_error.done():
+ self._callback_error.set_result(exc)
+ await asyncio.sleep(0)
+ except BaseException as exc:
+ if not self._callback_error.done():
+ self._callback_error.set_result(exc)
+ await asyncio.sleep(0)
+
+ delivery = asyncio.create_task(invoke_callback())
+ self._active_deliveries.add(delivery)
+
+ def retire_delivery(completed: asyncio.Task[None]) -> None:
+ self._active_deliveries.discard(completed)
+ with contextlib.suppress(BaseException):
+ completed.result()
+
+ delivery.add_done_callback(retire_delivery)
+ try:
+ await asyncio.shield(delivery)
+ except asyncio.CancelledError:
+ # Cancellation of the collector/finalizer must explicitly target
+ # the shielded callback task. Repeating cancellation handles a
+ # callback that performs async cleanup after its first cancel.
+ self._delivery_cancelled = True
+ cancellation = asyncio.create_task(_cancel_task_repeatedly(delivery, timeout=_REAP_CANCEL_SECONDS))
+ await _await_task_uninterruptibly(cancellation, preserve_cancellation=False)
+ raise
+
+ async def feed(self, text: str, stream: OutputStream) -> None:
+ await self._emit(self._redactors[stream].feed(text), stream)
+
+ async def finish(self, *, stderr_suffix: str = "") -> None:
+ await self._emit(self._redactors["stdout"].finish(), "stdout")
+ if stderr_suffix:
+ await self._emit(self._redactors["stderr"].feed(stderr_suffix), "stderr")
+ await self._emit(self._redactors["stderr"].finish(), "stderr")
+
+
async def _await_task_uninterruptibly(
task: asyncio.Task[Any],
*,
@@ -147,6 +263,27 @@ async def _await_task_uninterruptibly(
return result
+async def _cancel_task_repeatedly(task: asyncio.Future[Any], *, timeout: float) -> bool:
+ """Bound cleanup even when a coroutine suppresses its first cancellation."""
+ deadline = asyncio.get_running_loop().time() + timeout
+ retry_interval = min(0.01, timeout / 4) if timeout > 0 else 0
+ while not task.done():
+ task.cancel()
+ # Give the cancellation target a chance to catch the injected error
+ # before deciding whether another cancellation is necessary.
+ await asyncio.sleep(0)
+ if task.done():
+ break
+ remaining = deadline - asyncio.get_running_loop().time()
+ if remaining <= 0:
+ break
+ await asyncio.wait({task}, timeout=min(retry_interval, remaining))
+ if task.done():
+ with contextlib.suppress(asyncio.CancelledError, Exception):
+ task.result()
+ return task.done()
+
+
def _looks_like_path_token(token: str) -> bool:
return token.startswith(("/", "~/", "$"))
@@ -357,6 +494,15 @@ def __init__(
asyncio.subprocess.Process,
asyncio.Task[tuple[bytes, bytes]] | None,
] = {}
+ self._active_process_secret_values: dict[asyncio.subprocess.Process, set[str]] = {}
+ self._pending_creations: set[asyncio.Task[asyncio.subprocess.Process]] = set()
+ self._creation_secret_values: dict[asyncio.Task[asyncio.subprocess.Process], set[str]] = {}
+ self._creation_cleanups: dict[
+ asyncio.Task[asyncio.subprocess.Process],
+ asyncio.Task[None],
+ ] = {}
+ self._creation_cleanup_errors: list[set[str]] = []
+ self._stop_requested = False
super().__init__(*args, **kwargs)
base_dir = self._working_dir_override or (self.trial_paths.trial_dir / "local-environment")
self._root = base_dir.resolve()
@@ -389,6 +535,16 @@ def _validate_definition(self) -> None:
async def start(self, force_build: bool = False) -> None:
_ = force_build
+ if (
+ self._pending_creations
+ or self._creation_secret_values
+ or self._creation_cleanups
+ or self._creation_cleanup_errors
+ or self._active_processes
+ or self._active_process_secret_values
+ ):
+ raise RuntimeError("Cannot start local environment while process cleanup is still pending")
+ self._stop_requested = False
self.trial_paths.mkdir()
for path in (
self._root,
@@ -418,12 +574,142 @@ async def start(self, force_build: bool = False) -> None:
self.logger.warning("local mode is NOT kernel-sandboxed; advisory guardrails only: %s", plan.reason)
async def stop(self, delete: bool) -> None:
+ self._stop_requested = True
+ pending_creations = tuple(self._pending_creations)
+ for creation in pending_creations:
+ self._schedule_creation_cleanup(creation)
+ if pending_creations:
+ await asyncio.wait(
+ pending_creations,
+ timeout=_CREATION_CANCEL_SECONDS,
+ )
+
+ async def wait_for_resolved_creation_cleanups() -> None:
+ cleanup_tasks = tuple(
+ cleanup
+ for creation, cleanup in self._creation_cleanups.items()
+ if creation.done() and not cleanup.done()
+ )
+ if cleanup_tasks:
+ await asyncio.wait(cleanup_tasks, timeout=_STOP_CLEANUP_SECONDS)
+ # Let cleanup callbacks retire their creation/mapping entries
+ # before the final containment check.
+ await asyncio.sleep(0)
+
+ await wait_for_resolved_creation_cleanups()
+
for proc, communication in tuple(self._active_processes.items()):
- await self._terminate_process_tree(proc, communication)
- self._active_processes.clear()
+ try:
+ await self._terminate_process_tree(proc, communication)
+ except BaseException:
+ # Keep the handle and protected values for a redacted retry.
+ continue
+ self._release_active_process(proc)
+
+ # A pending creation can resolve while known active processes are
+ # being reaped. Give its already-tracked cleanup the same bounded
+ # opportunity before deciding whether deletion is safe.
+ await wait_for_resolved_creation_cleanups()
+
+ if (
+ not self._pending_creations
+ and not self._creation_secret_values
+ and not self._creation_cleanups
+ and not self._active_processes
+ ):
+ # Every process whose earlier cleanup failed has now been reaped
+ # through its retained active-process handle.
+ self._creation_cleanup_errors.clear()
+
+ unresolved_creations = tuple(creation for creation in self._pending_creations if not creation.done())
+ outstanding_cleanups = tuple(cleanup for cleanup in self._creation_cleanups.values() if not cleanup.done())
+ if (
+ unresolved_creations
+ or self._creation_secret_values
+ or outstanding_cleanups
+ or self._creation_cleanup_errors
+ or self._active_processes
+ ):
+ details: list[str] = []
+ if unresolved_creations:
+ details.append("process creation containment remains unresolved")
+ if outstanding_cleanups:
+ details.append("late process cleanup remains unresolved")
+ if self._creation_cleanup_errors:
+ details.append("creation cleanup failed")
+ if self._active_processes:
+ details.append("active process cleanup remains unresolved")
+ diagnostic = "Local environment stop could not confirm process creation containment before the deadline" + (
+ f" ({'; '.join(details)})" if details else ""
+ )
+ cleanup_secret_values = set().union(
+ *self._creation_cleanup_errors,
+ *self._creation_secret_values.values(),
+ *self._active_process_secret_values.values(),
+ )
+ raise RuntimeError(self._redact_output(diagnostic, cleanup_secret_values))
+
if delete and self._root.exists():
shutil.rmtree(self._root, ignore_errors=True)
+ def _schedule_creation_cleanup(
+ self,
+ creation: asyncio.Task[asyncio.subprocess.Process],
+ *,
+ secret_values: set[str] | None = None,
+ ) -> asyncio.Task[None]:
+ if secret_values:
+ self._creation_secret_values.setdefault(creation, set()).update(secret_values)
+ existing = self._creation_cleanups.get(creation)
+ if existing is not None:
+ return existing
+
+ async def reap_created_process() -> None:
+ try:
+ process = await asyncio.shield(creation)
+ except BaseException:
+ return
+ communication = asyncio.create_task(process.communicate())
+ self._active_processes[process] = communication
+ self._active_process_secret_values[process] = set(self._creation_secret_values.get(creation, set()))
+ try:
+ await self._terminate_process_tree(process, communication)
+ except BaseException:
+ # Retain ownership so stop() can retry containment.
+ raise
+ self._release_active_process(process)
+
+ cleanup = asyncio.create_task(reap_created_process())
+ self._creation_cleanups[creation] = cleanup
+
+ def finish_cleanup(completed: asyncio.Task[None]) -> None:
+ self._pending_creations.discard(creation)
+ self._creation_cleanups.pop(creation, None)
+ protected_values = self._creation_secret_values.pop(creation, set())
+ try:
+ completed.result()
+ except BaseException:
+ self._creation_cleanup_errors.append(protected_values)
+ diagnostic = self._redact_output(
+ "Local process creation cleanup failed",
+ protected_values,
+ )
+ with contextlib.suppress(RuntimeError):
+ loop = asyncio.get_running_loop()
+ loop.call_exception_handler(
+ {
+ "message": diagnostic,
+ "exception": RuntimeError(diagnostic),
+ }
+ )
+
+ cleanup.add_done_callback(finish_cleanup)
+ return cleanup
+
+ def _release_active_process(self, process: asyncio.subprocess.Process) -> None:
+ self._active_processes.pop(process, None)
+ self._active_process_secret_values.pop(process, None)
+
async def prepare_logs_for_host(self) -> None:
return None
@@ -474,35 +760,35 @@ async def exec(
user: str | int | None = None,
) -> ExecResult:
_ = user
+ output_callback = self._output_callback()
+ output_secret_values = self._output_secret_values(env, {})
+
+ async def blocked(reason: str) -> ExecResult:
+ diagnostic = self._redact_output(f"Local mode command blocked: {reason}", output_secret_values)
+ self.logger.warning("%s", diagnostic)
+ if output_callback is not None:
+ await output_callback(diagnostic, "stderr")
+ return ExecResult(stdout="", stderr=diagnostic, return_code=126)
+
+ if self._stop_requested:
+ return await blocked("environment shutdown is in progress")
+
rewritten = self._rewrite_command(command)
try:
workdir = self._resolve_path(cwd) if cwd else self._workspace
except ValueError as exc:
- return ExecResult(stdout="", stderr=f"Local mode command blocked: {exc}", return_code=126)
+ return await blocked(str(exc))
if not self._path_is_within_allowed_local_roots(workdir):
- return ExecResult(
- stdout="",
- stderr=f"Local mode command blocked: cwd {workdir} is outside the local run directory.",
- return_code=126,
- )
+ return await blocked(f"cwd {workdir} is outside the local run directory.")
workdir.mkdir(parents=True, exist_ok=True)
try:
exec_env = self._exec_env(env)
except ValueError as exc:
- self.logger.warning("Local mode command blocked: %s", exc)
- return ExecResult(
- stdout="",
- stderr=f"Local mode command blocked: {exc}",
- return_code=126,
- )
+ return await blocked(str(exc))
+ output_secret_values.update(self._output_secret_values(env, exec_env))
guardrail_reason = self._local_command_guardrail_reason(command, rewritten, exec_env)
if guardrail_reason:
- self.logger.warning("Local mode command blocked: %s", guardrail_reason)
- return ExecResult(
- stdout="",
- stderr=f"Local mode command blocked: {guardrail_reason}",
- return_code=126,
- )
+ return await blocked(guardrail_reason)
sandbox = self._sandbox
if sandbox is None:
@@ -551,43 +837,296 @@ async def exec(
start_new_session=os.name == "posix",
)
)
+ self._pending_creations.add(creation)
+ self._creation_secret_values[creation] = set(output_secret_values)
try:
proc = await asyncio.shield(creation)
- except asyncio.CancelledError:
- # A second cancellation must not propagate into ``creation`` after
- # the OS process exists but before asyncio returns its handle.
- proc = await _await_task_uninterruptibly(creation, preserve_cancellation=False)
- await self._terminate_process_tree(proc)
+ except asyncio.CancelledError as primary_error:
+ # Cancellation may arrive after the OS process exists but before
+ # asyncio returns its handle. Bound how long the caller waits for
+ # an uncooperative creation coroutine. The tracked cleanup owns
+ # the eventual handle and remains visible to stop().
+ cleanup = self._schedule_creation_cleanup(creation)
+ resolution = asyncio.create_task(asyncio.wait({creation}, timeout=_CREATION_CANCEL_SECONDS))
+ done, _pending = await _await_task_uninterruptibly(
+ resolution,
+ preserve_cancellation=False,
+ )
+ if creation in done:
+ safe_cleanup_error: RuntimeError | None = None
+ try:
+ await _await_task_uninterruptibly(cleanup, preserve_cancellation=False)
+ except BaseException as cleanup_error:
+ safe_cleanup_error = self._redacted_cleanup_error(cleanup_error, output_secret_values)
+ if safe_cleanup_error is not None:
+ self._raise_primary_with_cleanup(
+ primary_error,
+ safe_cleanup_error,
+ output_secret_values,
+ note_prefix="Local process-tree cleanup also failed during creation cancellation",
+ )
+ else:
+ primary_error.add_note(
+ self._redact_output(
+ "Local process creation cancellation remained pending past the cleanup deadline; "
+ "a tracked late-process reaper remains active",
+ output_secret_values,
+ )
+ )
raise
- self._active_processes[proc] = None
- communication = asyncio.create_task(proc.communicate(input=env_payload))
+ except BaseException:
+ self._pending_creations.discard(creation)
+ self._creation_secret_values.pop(creation, None)
+ raise
+
+ if self._stop_requested:
+ cleanup = self._schedule_creation_cleanup(creation)
+ safe_cleanup_error: RuntimeError | None = None
+ try:
+ await _await_task_uninterruptibly(cleanup, preserve_cancellation=False)
+ except BaseException as cleanup_error:
+ safe_cleanup_error = self._redacted_cleanup_error(cleanup_error, output_secret_values)
+ if safe_cleanup_error is not None:
+ raise safe_cleanup_error from None
+ raise RuntimeError("Local process creation completed during environment shutdown")
+ self._pending_creations.discard(creation)
+ self._creation_secret_values.pop(creation, None)
+ callback_error: asyncio.Future[BaseException] | None = None
+ callback_output: _StreamCallbackOutput | None = None
+ if output_callback is not None:
+ callback_error = asyncio.get_running_loop().create_future()
+ callback_output = _StreamCallbackOutput(output_callback, callback_error, output_secret_values)
+ communication = asyncio.create_task(
+ self._collect_streamed_output(
+ proc,
+ env_payload,
+ callback_output,
+ )
+ )
self._active_processes[proc] = communication
+ self._active_process_secret_values[proc] = set(output_secret_values)
+ process_contained = False
+
+ async def terminate_preserving_primary(primary_error: BaseException) -> tuple[bytes, bytes]:
+ nonlocal process_contained
+ cleanup = asyncio.create_task(self._terminate_process_tree(proc, communication))
+ safe_cleanup_error: RuntimeError | None = None
+ try:
+ result = await _await_task_uninterruptibly(cleanup, preserve_cancellation=False)
+ except BaseException as cleanup_error:
+ safe_cleanup_error = self._redacted_cleanup_error(cleanup_error, output_secret_values)
+ if callback_output is not None and not communication.done():
+ callback_output.abandon_delivery()
+ if safe_cleanup_error is not None:
+ self._raise_primary_with_cleanup(
+ primary_error,
+ safe_cleanup_error,
+ output_secret_values,
+ note_prefix="Local process-tree cleanup also failed",
+ )
+ process_contained = True
+ return result
+
+ async def cleanup_process_tree() -> tuple[bytes, bytes]:
+ """Contain the group while preserving cancellation as primary."""
+ nonlocal process_contained
+ cleanup = asyncio.create_task(self._terminate_process_tree(proc, communication))
+ safe_cleanup_error: RuntimeError | None = None
+ try:
+ result = await asyncio.shield(cleanup)
+ except asyncio.CancelledError as primary_error:
+ try:
+ result = await _await_task_uninterruptibly(cleanup, preserve_cancellation=False)
+ except BaseException as cleanup_error:
+ safe_cleanup_error = self._redacted_cleanup_error(cleanup_error, output_secret_values)
+ if callback_output is not None and not communication.done():
+ callback_output.abandon_delivery()
+ if safe_cleanup_error is not None:
+ self._raise_primary_with_cleanup(
+ primary_error,
+ safe_cleanup_error,
+ output_secret_values,
+ note_prefix="Local process-tree cleanup also failed",
+ )
+ process_contained = True
+ self._release_active_process(proc)
+ raise
+ except BaseException as cleanup_error:
+ safe_cleanup_error = self._redacted_cleanup_error(cleanup_error, output_secret_values)
+ if callback_output is not None and not communication.done():
+ callback_output.abandon_delivery()
+ if safe_cleanup_error is not None:
+ raise safe_cleanup_error from None
+ process_contained = True
+ self._release_active_process(proc)
+ return result
+
+ callback_failure: BaseException | None = None
+ timed_out = False
try:
try:
- stdout_b, stderr_b = await asyncio.wait_for(
- asyncio.shield(communication),
+ waitables: set[asyncio.Future[Any] | asyncio.Task[Any]] = {communication}
+ if callback_error is not None:
+ waitables.add(callback_error)
+ done, _pending = await asyncio.wait(
+ waitables,
timeout=timeout_sec,
+ return_when=asyncio.FIRST_COMPLETED,
)
- except TimeoutError:
- stdout_b, stderr_b = await self._terminate_process_tree(proc, communication)
- stdout = self._redact_output(stdout_b.decode(errors="replace"), exec_env)
- stderr = self._redact_output(stderr_b.decode(errors="replace"), exec_env)
+ if not done:
+ timed_out = True
+ elif callback_error is not None and callback_error in done:
+ callback_failure = callback_error.result()
+ else:
+ stdout_b, stderr_b = await asyncio.shield(communication)
+ except asyncio.CancelledError as primary_error:
+ await terminate_preserving_primary(primary_error)
+ raise
+ except BaseException as primary_error:
+ await terminate_preserving_primary(primary_error)
+ raise
+
+ if callback_failure is None and callback_error is not None and callback_error.done():
+ callback_failure = callback_error.result()
+ if callback_failure is not None:
+ await terminate_preserving_primary(callback_failure)
+ raise callback_failure
+
+ if timed_out:
+ stdout_b, stderr_b = await cleanup_process_tree()
+ if callback_output is not None:
+ stdout_b = callback_output.raw_output("stdout")
+ stderr_b = callback_output.raw_output("stderr")
+ if callback_error is not None and callback_error.done():
+ callback_failure = callback_error.result()
+ raise callback_failure
+ raw_stdout = stdout_b.decode(errors="replace")
+ raw_stderr = stderr_b.decode(errors="replace")
+ diagnostic = "Timed out" if not raw_stderr or raw_stderr.endswith("\n") else "\nTimed out"
+ if callback_output is not None:
+ callback_finish = asyncio.create_task(callback_output.finish(stderr_suffix=diagnostic))
+
+ async def cancel_callback_finish(*, preserve_cancellation: bool) -> bool:
+ callback_output.abandon_delivery()
+ cancellation = asyncio.create_task(
+ _cancel_task_repeatedly(callback_finish, timeout=_REAP_CANCEL_SECONDS)
+ )
+ return bool(
+ await _await_task_uninterruptibly(
+ cancellation,
+ preserve_cancellation=preserve_cancellation,
+ )
+ )
+
+ try:
+ done, _pending = await asyncio.wait(
+ {callback_finish},
+ timeout=_REAP_CANCEL_SECONDS,
+ )
+ except asyncio.CancelledError:
+ await cancel_callback_finish(preserve_cancellation=False)
+ raise
+ if callback_finish not in done:
+ if not await cancel_callback_finish(preserve_cancellation=True):
+ raise RuntimeError(
+ self._redact_output(
+ "Local callback cleanup remained pending past the deadline",
+ output_secret_values,
+ )
+ )
+ else:
+ callback_finish.result()
+ if callback_error is not None and callback_error.done():
+ raise callback_error.result()
+ stdout = self._redact_output(raw_stdout, output_secret_values)
+ timeout_stderr = self._redact_output(raw_stderr + diagnostic, output_secret_values)
return ExecResult(
stdout=stdout,
- stderr=(stderr + "\nTimed out").strip(),
+ stderr=timeout_stderr,
return_code=124,
)
- except asyncio.CancelledError:
- await self._terminate_process_tree(proc, communication)
- raise
+
+ # proc.wait()/communicate() only proves that the launcher exited;
+ # a same-group descendant may have closed its inherited streams
+ # and survived. Contain the group immediately, before an
+ # arbitrarily slow final callback creates a PID-reuse window.
+ await cleanup_process_tree()
+
+ if callback_output is not None:
+ await callback_output.finish()
+ if callback_error is not None and callback_error.done():
+ callback_failure = callback_error.result()
+ raise callback_failure
return ExecResult(
- stdout=self._redact_output(stdout_b.decode(errors="replace"), exec_env),
- stderr=self._redact_output(stderr_b.decode(errors="replace"), exec_env),
+ stdout=self._redact_output(stdout_b.decode(errors="replace"), output_secret_values),
+ stderr=self._redact_output(stderr_b.decode(errors="replace"), output_secret_values),
return_code=int(proc.returncode or 0),
)
finally:
- self._active_processes.pop(proc, None)
+ if process_contained:
+ self._release_active_process(proc)
+
+ @staticmethod
+ async def _collect_streamed_output(
+ proc: asyncio.subprocess.Process,
+ stdin_data: bytes,
+ callback_output: _StreamCallbackOutput | None,
+ ) -> tuple[bytes, bytes]:
+ """Drain both streams within one combined hard raw-byte budget."""
+ if proc.stdin is None or proc.stdout is None or proc.stderr is None:
+ raise RuntimeError("local subprocess pipe invariant violated")
+ output_budget = CommandOutputByteBudget()
+
+ async def write_stdin() -> None:
+ try:
+ proc.stdin.write(stdin_data)
+ await proc.stdin.drain()
+ except (BrokenPipeError, ConnectionResetError):
+ # Match asyncio's communicate(): an early child exit is
+ # represented by its return code, not a host-side pipe error.
+ pass
+ finally:
+ proc.stdin.close()
+ with contextlib.suppress(BrokenPipeError, ConnectionResetError):
+ await proc.stdin.wait_closed()
+
+ async def drain_stream(
+ reader: asyncio.StreamReader,
+ stream: OutputStream,
+ ) -> bytes:
+ decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+ output = bytearray()
+
+ while chunk := await reader.read(64 * 1024):
+ output_budget.consume(chunk)
+ output.extend(chunk)
+ if callback_output is not None:
+ callback_output.append_raw(chunk, stream)
+ if text := decoder.decode(chunk):
+ await callback_output.feed(text, stream)
+ # A callback coroutine is allowed to complete synchronously;
+ # still give command timeout/cancellation and the other stream
+ # a scheduling point after each bounded read.
+ await asyncio.sleep(0)
+ if callback_output is not None and (text := decoder.decode(b"", final=True)):
+ await callback_output.feed(text, stream)
+ return bytes(output)
+
+ stdin_task = asyncio.create_task(write_stdin())
+ stdout_task = asyncio.create_task(drain_stream(proc.stdout, "stdout"))
+ stderr_task = asyncio.create_task(drain_stream(proc.stderr, "stderr"))
+ wait_task = asyncio.create_task(proc.wait())
+ tasks = (stdin_task, stdout_task, stderr_task, wait_task)
+ try:
+ await asyncio.gather(*tasks)
+ except BaseException:
+ for task in tasks:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ raise
+ return stdout_task.result(), stderr_task.result()
async def exec_with_sensitive_env(
self,
@@ -615,35 +1154,93 @@ async def reap() -> tuple[bytes, bytes]:
active_communication = communication
if active_communication is None:
active_communication = asyncio.create_task(proc.communicate())
+ process_wait: asyncio.Task[tuple[bytes, bytes]] | None = None
def send(sig: signal.Signals) -> None:
if os.name == "posix":
- with contextlib.suppress(ProcessLookupError):
+ try:
os.killpg(proc.pid, sig)
+ except ProcessLookupError:
+ return
+ except PermissionError:
+ if proc.returncode is not None:
+ return
+ try:
+ os.getpgid(proc.pid)
+ except ProcessLookupError:
+ return
+ raise
elif proc.returncode is None:
if sig == signal.SIGTERM:
proc.terminate()
else:
proc.kill()
+ async def wait_for_process_group_exit(seconds: float) -> None:
+ if os.name != "posix" or not isinstance(proc, asyncio.subprocess.Process):
+ return
+ deadline = asyncio.get_running_loop().time() + seconds
+ while True:
+ try:
+ os.killpg(proc.pid, 0)
+ except ProcessLookupError:
+ return
+ except PermissionError:
+ return
+ remaining = deadline - asyncio.get_running_loop().time()
+ if remaining <= 0:
+ return
+ await asyncio.sleep(min(0.01, remaining))
+
async def bounded_wait(seconds: float) -> tuple[bytes, bytes] | None:
+ nonlocal process_wait
try:
return await asyncio.wait_for(asyncio.shield(active_communication), timeout=seconds)
except TimeoutError:
return None
+ except BaseException:
+ # A collector failure is the caller's primary error, but
+ # cleanup still has to confirm process exit. Fall back to
+ # waiting on the process handle without re-reading pipes.
+ if not hasattr(proc, "wait"):
+ raise
+ if process_wait is None:
+
+ async def wait_for_process() -> tuple[bytes, bytes]:
+ await proc.wait()
+ return b"", b""
+
+ process_wait = asyncio.create_task(wait_for_process())
+ try:
+ return await asyncio.wait_for(asyncio.shield(process_wait), timeout=seconds)
+ except TimeoutError:
+ return None
send(signal.SIGTERM)
- if output := await bounded_wait(_REAP_TERM_SECONDS):
- return output
+ term_output = await bounded_wait(_REAP_TERM_SECONDS)
+ if os.name != "posix" and term_output is not None:
+ return term_output
+
+ # On POSIX the launcher can exit and close its pipes while a
+ # same-group descendant ignores SIGTERM. Escalate to the original
+ # group immediately even when communication already completed;
+ # this minimizes the process-group-ID reuse window and prevents a
+ # successful collector from being mistaken for containment.
send(signal.SIGKILL)
- if output := await bounded_wait(_REAP_KILL_SECONDS):
- return output
-
- active_communication.cancel()
- done, _pending = await asyncio.wait({active_communication}, timeout=_REAP_CANCEL_SECONDS)
- if active_communication in done:
+ group_exit = asyncio.create_task(wait_for_process_group_exit(_REAP_KILL_SECONDS))
+ kill_output = term_output
+ if kill_output is None:
+ kill_output = await bounded_wait(_REAP_KILL_SECONDS)
+ await group_exit
+ if kill_output is not None:
+ return kill_output
+
+ if not await _cancel_task_repeatedly(active_communication, timeout=_REAP_CANCEL_SECONDS):
+ raise RuntimeError("Local output cleanup remained pending past the reap deadline")
+ if process_wait is not None and not process_wait.done():
+ process_wait.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
- active_communication.result()
+ await process_wait
return b"", b""
cleanup = asyncio.create_task(reap())
@@ -981,12 +1578,60 @@ def _is_relative_to(path: Path, root: Path) -> bool:
except ValueError:
return False
+ def _output_secret_values(self, env: dict[str, str] | None, exec_env: dict[str, str]) -> set[str]:
+ merged = self._merge_env(env) or {}
+ secret_values = {
+ value
+ for key, value in merged.items()
+ if value
+ and (
+ _SENSITIVE_ENV_NAME_RE.search(key)
+ or _COMPACT_SENSITIVE_ENV_NAME_RE.search(key)
+ or (len(value) >= _MIN_LEGACY_SECRET_VALUE_LENGTH and _LEGACY_SECRET_ENV_NAME_RE.search(key))
+ )
+ }
+ secret_values.update(
+ value
+ for key, value in exec_env.items()
+ if value
+ and (
+ _SENSITIVE_ENV_NAME_RE.search(key)
+ or _COMPACT_SENSITIVE_ENV_NAME_RE.search(key)
+ or (len(value) >= _MIN_LEGACY_SECRET_VALUE_LENGTH and _LEGACY_SECRET_ENV_NAME_RE.search(key))
+ )
+ )
+ secret_values.update(_credential_uri_environment_values(merged))
+ secret_values.update(_credential_uri_environment_values(exec_env))
+ return secret_values
+
@staticmethod
- def _redact_output(text: str, env: dict[str, str]) -> str:
- secret_values = [
- value for key, value in env.items() if _SECRET_ENV_NAME_RE.search(key) and value and len(value) >= 8
- ]
- return redact_secrets_in_log_line(text, extra_secret_values=secret_values)
+ def _redact_output(text: str, secret_values: set[str]) -> str:
+ redactor = StreamingLogRedactor(secret_values)
+ return redactor.feed(text) + redactor.finish()
+
+ def _redacted_cleanup_error(self, error: BaseException, secret_values: set[str]) -> RuntimeError:
+ """Return a fresh cleanup error with no reference to the raw exception."""
+ error_type = type(error).__name__
+ try:
+ summary = f"{error_type}: {error}"
+ except BaseException:
+ summary = f"{error_type}: cleanup detail unavailable"
+ return RuntimeError(self._redact_output(summary, secret_values))
+
+ def _raise_primary_with_cleanup(
+ self,
+ primary_error: BaseException,
+ cleanup_error: RuntimeError,
+ secret_values: set[str],
+ *,
+ note_prefix: str,
+ ) -> NoReturn:
+ note = self._redact_output(
+ f"{note_prefix}: {cleanup_error}",
+ secret_values,
+ )
+ primary_error.add_note(note)
+ raise primary_error from cleanup_error
def _path_with_evaluator_python(self, path: str) -> str:
"""Ensure local verifier scripts use the evaluator's Python runtime."""
@@ -1013,6 +1658,11 @@ def _filter_command_env(env: dict[str, str], *, protected: set[str]) -> dict[str
for key, value in env.items():
normalized = key.upper()
if normalized in _BLOCKED_COMMAND_ENV_NAMES or normalized.startswith(_BLOCKED_COMMAND_ENV_PREFIXES):
+ # Docker tasks deliberately reset loader-controlled variables
+ # to the empty string. In local mode, absence is the safer
+ # equivalent; reject every non-empty value and drop the reset.
+ if value == "":
+ continue
raise ValueError(
f"environment variable {key} can execute or alter code before confinement and is not allowed"
)
diff --git a/src/skillevaluator/tier3/harbor/metrics.py b/src/skillevaluator/tier3/harbor/metrics.py
index 98ed8658..7577a581 100644
--- a/src/skillevaluator/tier3/harbor/metrics.py
+++ b/src/skillevaluator/tier3/harbor/metrics.py
@@ -6,9 +6,11 @@
from __future__ import annotations
import math
+import re
from typing import Any
from skillevaluator.constants import DIMENSION_MAPPING
+from skillevaluator.utils.redaction import contains_credential_value, is_sensitive_key
DEFAULT_METRIC_SET = "skill-evaluator-default-v2"
LEGACY_METRIC_SET = "skill-evaluator-default-v1"
@@ -83,6 +85,8 @@
}
_RESERVED_METADATA_KEYS = {
+ "custom_details",
+ "custom_metrics",
"details",
"entry_id",
"error",
@@ -99,17 +103,142 @@
RESERVED_METRIC_NAMES = frozenset(DEFAULT_METRICS) | _RESERVED_METADATA_KEYS
+# Keep collection, aggregation, generated JSON, and the canonical report on one
+# explicit custom-metric envelope. The report visits at most 128 names per
+# reward; enforcing the same limit before aggregation prevents summaries and
+# paired lift artifacts from expanding past their browser-safe bounds.
+MAX_CUSTOM_METRICS = 128
+MAX_CUSTOM_METRIC_NAME_BYTES = 256
+
+_SAFE_SENSITIVE_METRIC_PREFIXES = {"auth", "secret", "token"}
+_SAFE_SENSITIVE_METRIC_SUFFIXES = {
+ "accuracy",
+ "compliance",
+ "count",
+ "coverage",
+ "efficiency",
+ "handling",
+ "leakage",
+ "precision",
+ "quality",
+ "rate",
+ "ratio",
+ "recall",
+ "safety",
+ "score",
+ "usage",
+}
+
+
+class CustomMetricContractError(ValueError):
+ """Raised when custom metrics exceed the bounded publication contract."""
+
+
+def _custom_metric_name_shape_is_valid(name: str) -> bool:
+ try:
+ encoded = name.encode("utf-8")
+ except UnicodeError:
+ return False
+ return bool(name) and name == name.strip() and name.isprintable() and len(encoded) <= MAX_CUSTOM_METRIC_NAME_BYTES
+
+
+def _is_explicit_safe_sensitive_metric_name(name: str) -> bool:
+ camel_split = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
+ normalized = re.sub(r"[^a-zA-Z0-9]+", "_", camel_split).strip("_").lower()
+ parts = tuple(part for part in normalized.split("_") if part)
+ return (
+ len(parts) == 2 and parts[0] in _SAFE_SENSITIVE_METRIC_PREFIXES and parts[1] in _SAFE_SENSITIVE_METRIC_SUFFIXES
+ )
+
+
+def _custom_metric_name_contains_sensitive_data(name: str) -> bool:
+ return bool(
+ contains_credential_value(name)
+ or (is_sensitive_key(name) and not _is_explicit_safe_sensitive_metric_name(name))
+ )
+
+
+def custom_metric_name_is_publishable(name: object) -> bool:
+ """Return whether a custom metric name is bounded and safe to publish."""
+ text = str(name)
+ if not _custom_metric_name_shape_is_valid(text):
+ return False
+ # Credential values can themselves appear as keys. Never turn those into a
+ # visible ```` alias because aliases can collide. Narrow,
+ # explicitly metric-shaped names such as ``secret_handling`` remain valid.
+ return not _custom_metric_name_contains_sensitive_data(text)
+
+
+def _iter_custom_metric_candidates(reward: dict[str, Any]):
+ explicit = reward.get("custom_metrics")
+ if isinstance(explicit, dict):
+ yield from explicit.items()
+
+ metrics = reward.get("metrics")
+ if isinstance(metrics, dict):
+ yield from metrics.items()
+
+ for name, value in reward.items():
+ if str(name) in RESERVED_METRIC_NAMES or str(name).startswith("_"):
+ continue
+ yield name, value
+
+
+def custom_metric_contract_error(reward: dict[str, Any]) -> str | None:
+ """Return a fixed diagnostic when one reward exceeds the custom-metric contract."""
+ explicit = reward.get("custom_metrics")
+ if "custom_metrics" in reward and not isinstance(explicit, dict):
+ return "Custom metrics container must be a JSON object"
+ if isinstance(explicit, dict) and any(str(name) in RESERVED_METRIC_NAMES for name in explicit):
+ return "Custom metric collides with reserved SkillEvaluator metric names"
+
+ names: set[str] = set()
+ for raw_name, raw_value in _iter_custom_metric_candidates(reward):
+ name = str(raw_name)
+ if name in RESERVED_METRIC_NAMES:
+ continue
+ value = raw_value.get("score") if isinstance(raw_value, dict) else raw_value
+ if score_value(value) is None:
+ continue
+ # Secret-shaped names are deliberately omitted rather than reported or
+ # aliased. Their content must not reach diagnostics either.
+ if _custom_metric_name_contains_sensitive_data(name):
+ continue
+ if not _custom_metric_name_shape_is_valid(name):
+ return "Custom metric name exceeds the bounded publication contract"
+ names.add(name)
+ if len(names) > MAX_CUSTOM_METRICS:
+ return "Custom metric count exceeds the per reward publication limit"
+ return None
+
def _finite_number(value: object) -> float | None:
if not isinstance(value, int | float) or isinstance(value, bool):
return None
- numeric = float(value)
+ try:
+ numeric = float(value)
+ except (OverflowError, ValueError):
+ return None
return numeric if math.isfinite(numeric) else None
+def score_value(value: object) -> float | None:
+ """Return a finite score inside SkillEvaluator's documented 0..1 range."""
+ numeric = _finite_number(value)
+ return numeric if numeric is not None and 0.0 <= numeric <= 1.0 else None
+
+
+def _has_metric_field(reward: dict[str, Any], metric: str) -> bool:
+ """Return whether a reward claims a metric, independently of score validity."""
+ if metric in reward:
+ return True
+ metrics = reward.get("metrics")
+ return isinstance(metrics, dict) and metric in metrics
+
+
def metric_value(reward: dict[str, Any], metric: str) -> float | None:
"""Return a numeric metric value from a reward payload, if present."""
- val = _finite_number(reward.get(metric))
+ val = score_value(reward.get(metric))
if val is not None:
return val
@@ -118,7 +247,7 @@ def metric_value(reward: dict[str, Any], metric: str) -> float | None:
raw = metrics.get(metric)
if isinstance(raw, dict):
raw = raw.get("score")
- numeric = _finite_number(raw)
+ numeric = score_value(raw)
if numeric is not None:
return numeric
@@ -132,16 +261,32 @@ def metric_set_for_reward(reward: dict[str, Any]) -> tuple[str, tuple[str, ...]]
return DEFAULT_METRIC_SET, DEFAULT_METRICS
if metric_set == LEGACY_METRIC_SET:
return LEGACY_METRIC_SET, LEGACY_METRICS
+ if metric_set:
+ # Explicit custom metric sets cannot claim SkillEvaluator's reserved
+ # canonical names. Their only standard score surface is ``overall``.
+ return CUSTOM_ONLY_METRIC_SET, ()
- if metric_value(reward, "security") is not None:
+ if _has_metric_field(reward, "security"):
return DEFAULT_METRIC_SET, DEFAULT_METRICS
- if any(metric_value(reward, m) is not None for m in LEGACY_METRICS):
+ if any(_has_metric_field(reward, metric) for metric in LEGACY_METRICS):
return LEGACY_METRIC_SET, LEGACY_METRICS
- if _finite_number(reward.get("overall")) is not None:
+ if score_value(reward.get("overall")) is not None:
return CUSTOM_ONLY_METRIC_SET, ()
return DEFAULT_METRIC_SET, DEFAULT_METRICS
+def rewards_have_mixed_metric_contracts(rewards: object) -> bool:
+ """Return whether physical reward rows declare distinct metric contracts."""
+ if not isinstance(rewards, list):
+ return False
+ contracts = {
+ str(reward.get("metric_set") or reward.get("metric_set_version") or metric_set_for_reward(reward)[0])
+ for reward in rewards
+ if isinstance(reward, dict)
+ }
+ return len(contracts) > 1
+
+
def metric_set_for_rewards(rewards: list[dict[str, Any]]) -> tuple[str, tuple[str, ...]]:
"""Return the metric set for a collection, preferring the new SkillEvaluator set."""
declared = {str(reward.get("metric_set") or reward.get("metric_set_version") or "") for reward in rewards}
@@ -149,11 +294,14 @@ def metric_set_for_rewards(rewards: list[dict[str, Any]]) -> tuple[str, tuple[st
return DEFAULT_METRIC_SET, DEFAULT_METRICS
if LEGACY_METRIC_SET in declared:
return LEGACY_METRIC_SET, LEGACY_METRICS
- if any(metric_value(reward, "security") is not None for reward in rewards):
+ undeclared = [reward for reward in rewards if not (reward.get("metric_set") or reward.get("metric_set_version"))]
+ if any(_has_metric_field(reward, "security") for reward in undeclared):
return DEFAULT_METRIC_SET, DEFAULT_METRICS
- if any(any(metric_value(reward, m) is not None for m in LEGACY_METRICS) for reward in rewards):
+ if any(any(_has_metric_field(reward, metric) for metric in LEGACY_METRICS) for reward in undeclared):
return LEGACY_METRIC_SET, LEGACY_METRICS
- if any(_finite_number(reward.get("overall")) is not None for reward in rewards):
+ if any(score_value(reward.get("overall")) is not None for reward in rewards):
+ return CUSTOM_ONLY_METRIC_SET, ()
+ if any(value for value in declared):
return CUSTOM_ONLY_METRIC_SET, ()
return DEFAULT_METRIC_SET, DEFAULT_METRICS
@@ -167,7 +315,10 @@ def average_metrics(rewards: list[dict[str, Any]]) -> tuple[dict[str, float], st
metric_counts: dict[str, int] = dict.fromkeys(metrics, 0)
for reward in rewards:
+ _, reward_metrics = metric_set_for_reward(reward)
for metric in metrics:
+ if metric not in reward_metrics:
+ continue
val = metric_value(reward, metric)
if val is not None:
metric_sums[metric] += val
@@ -196,7 +347,7 @@ def overall_score(reward: dict[str, Any]) -> float | None:
return None
return sum(value for value in values if value is not None) / len(values)
- return _finite_number(reward.get("overall"))
+ return score_value(reward.get("overall"))
def score_definition(metrics: tuple[str, ...] = DEFAULT_METRICS) -> str:
@@ -210,7 +361,7 @@ def dimension_scores(scores: dict[str, float]) -> dict[str, dict[str, Any]]:
"""Compute report-only SkillEvaluator dimension scores from default metric scores."""
out: dict[str, dict[str, Any]] = {}
for dimension, sources in DIMENSION_DEFINITIONS.items():
- if not all(_finite_number(scores.get(metric)) is not None for metric in sources):
+ if not all(score_value(scores.get(metric)) is not None for metric in sources):
continue
total_weight = sum(sources.values())
if total_weight <= 0:
@@ -230,41 +381,50 @@ def extract_custom_metrics(reward: dict[str, Any]) -> dict[str, float]:
explicit = reward.get("custom_metrics")
if isinstance(explicit, dict):
for name, value in explicit.items():
- if name in RESERVED_METRIC_NAMES:
+ name = str(name)
+ if name in RESERVED_METRIC_NAMES or not custom_metric_name_is_publishable(name):
continue
if isinstance(value, dict):
value = value.get("score")
- numeric = _finite_number(value)
+ numeric = score_value(value)
if numeric is not None:
- custom[str(name)] = numeric
+ custom[name] = numeric
metrics = reward.get("metrics")
if isinstance(metrics, dict):
for name, value in metrics.items():
- if name in RESERVED_METRIC_NAMES:
+ name = str(name)
+ if name in RESERVED_METRIC_NAMES or not custom_metric_name_is_publishable(name):
continue
if isinstance(value, dict):
value = value.get("score")
- numeric = _finite_number(value)
+ numeric = score_value(value)
if numeric is not None:
- custom[str(name)] = numeric
+ custom[name] = numeric
for name, value in reward.items():
- if name in RESERVED_METRIC_NAMES or name.startswith("_"):
+ name = str(name)
+ if name in RESERVED_METRIC_NAMES or name.startswith("_") or not custom_metric_name_is_publishable(name):
continue
- numeric = _finite_number(value)
+ if isinstance(value, dict):
+ value = value.get("score")
+ numeric = score_value(value)
if numeric is not None:
- custom[str(name)] = numeric
+ custom[name] = numeric
return custom
def average_custom_metrics(rewards: list[dict[str, Any]]) -> dict[str, float]:
- """Average custom metrics across rewards."""
+ """Average custom metrics across rewards inside the publication envelope."""
sums: dict[str, float] = {}
counts: dict[str, int] = {}
for reward in rewards:
+ if reason := custom_metric_contract_error(reward):
+ raise CustomMetricContractError(reason)
for name, value in extract_custom_metrics(reward).items():
+ if name not in sums and len(sums) >= MAX_CUSTOM_METRICS:
+ raise CustomMetricContractError("Custom metric union exceeds the per condition publication limit")
sums[name] = sums.get(name, 0.0) + value
counts[name] = counts.get(name, 0) + 1
return {name: round(sums[name] / counts[name], 4) for name in sorted(sums)}
diff --git a/src/skillevaluator/tier3/harbor/progress.py b/src/skillevaluator/tier3/harbor/progress.py
index 37410af7..a4a3deef 100644
--- a/src/skillevaluator/tier3/harbor/progress.py
+++ b/src/skillevaluator/tier3/harbor/progress.py
@@ -20,6 +20,7 @@
from typing import Literal, Protocol, TextIO, runtime_checkable
from skillevaluator.tier3.harbor.secret_redaction import redact_secrets_in_log_line
+from skillevaluator.utils.redaction import credential_uri_secret_values
ProgressMode = Literal["auto", "rich", "plain", "off"]
logger = logging.getLogger(__name__)
@@ -32,6 +33,7 @@
r"(?:\"[^\"]*\"|'[^']*'|[^\s,;]+)"
)
_SECRET_ENV_NAME_RE = re.compile(r"(?i)(?:api[_-]?key|access[_-]?key|auth|credential|password|secret|token)")
+_CREDENTIAL_URI_USERINFO_RE = re.compile(r"(?i)(?P[a-z][a-z0-9+.-]{0,31}://)(?P[^\s/?#]+@)")
_ANSI_ESCAPE_RE = re.compile(r"\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
_OSC_ESCAPE_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
_TERMINAL_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
@@ -103,16 +105,41 @@ def redact_progress_detail(detail: object, *, secret_values: set[str] | None = N
text = _ANSI_ESCAPE_RE.sub("", text)
text = _TERMINAL_CONTROL_RE.sub("", text)
text = " ".join(text.split())
+ if "://" in text:
+ text = _CREDENTIAL_URI_USERINFO_RE.sub(r"\g@", text)
for secret in sorted(secret_values or (), key=len, reverse=True):
if len(secret) >= 4:
text = text.replace(secret, "")
+ elif secret:
+ # Exact credential-derived fragments can legitimately be short
+ # (for example proxy userinfo). Redact them only as standalone
+ # tokens so a one-character secret cannot erase normal prose.
+ text = re.sub(
+ rf"(?",
+ text,
+ )
text = redact_secrets_in_log_line(text, extra_secret_values=secret_values)
return _SECRET_ASSIGNMENT_RE.sub(r"\1", text)
def secret_values_from_environment(environment: Mapping[str, str]) -> set[str]:
"""Extract exact credential values without treating every env value as secret."""
- return {str(value) for name, value in environment.items() if value and _SECRET_ENV_NAME_RE.search(name)}
+ protected: set[str] = set()
+ for name, value in environment.items():
+ if not value:
+ continue
+ rendered = str(value)
+ if _SECRET_ENV_NAME_RE.search(name):
+ protected.add(rendered)
+ if name.upper().endswith("_PROXY") or "://" in rendered:
+ protected.update(
+ credential_uri_secret_values(
+ rendered,
+ allow_schemeless=name.upper().endswith("_PROXY"),
+ )
+ )
+ return protected
class PlainProgressReporter:
diff --git a/src/skillevaluator/tier3/harbor/report.py b/src/skillevaluator/tier3/harbor/report.py
index c2bb22d8..d47dd851 100644
--- a/src/skillevaluator/tier3/harbor/report.py
+++ b/src/skillevaluator/tier3/harbor/report.py
@@ -19,16 +19,53 @@
from skillevaluator.tier3.harbor import report_data
from skillevaluator.tier3.harbor.metrics import (
DEFAULT_METRICS,
+ MAX_CUSTOM_METRICS,
METRIC_DESCRIPTIONS,
METRIC_DISPLAY,
METRIC_QUESTIONS,
extract_custom_metrics,
+ metric_set_for_reward,
+ metric_value,
+ score_value,
)
+from skillevaluator.tier3.output_provenance import write_output_file_atomically
from skillevaluator.utils.redaction import redact_sensitive_data
logger = logging.getLogger(__name__)
DISPLAY_METRICS = DEFAULT_METRICS
+_MAX_FINDINGS = len(DEFAULT_METRICS) + MAX_CUSTOM_METRICS
+_MAX_FINDING_REASONS = 4
+_MAX_FINDING_EVIDENCE_REFS = 3
+_MAX_SUGGESTIONS = 4
+_MAX_FINDING_TEXT_CHARS = 2048
+_EVIDENCE_REF_TEXT_LIMITS = {
+ "source": 256,
+ "json_pointer": 512,
+ "kind": 64,
+ "path": 512,
+ "label": 256,
+ "excerpt": 512,
+}
+
+
+def _bounded_json_text(value: Any, *, max_encoded_bytes: int) -> str:
+ """Bound text by its ensure-ascii JSON representation, not Python characters."""
+ text = str(value or "")
+ if len(json.dumps(text, ensure_ascii=True).encode("utf-8")) - 2 <= max_encoded_bytes:
+ return text
+ suffix = "..."
+ low, high = 0, len(text)
+ while low < high:
+ midpoint = (low + high + 1) // 2
+ candidate = text[:midpoint] + suffix
+ encoded_size = len(json.dumps(candidate, ensure_ascii=True).encode("utf-8")) - 2
+ if encoded_size <= max_encoded_bytes:
+ low = midpoint
+ else:
+ high = midpoint - 1
+ return text[:low] + suffix if low else suffix[:max_encoded_bytes]
+
_METRIC_LABELS = {
"security": "SECURITY (unsafe operations, secret leakage, unauthorized access)",
@@ -80,6 +117,7 @@ def _load_trial_rewards(
def _pick_best_agent(
agents_data: dict[str, dict[str, Any]],
+ persisted_agents: dict[str, dict[str, Any]] | None = None,
) -> str:
"""Select the agent with the highest overall with-skill score."""
best_agent = ""
@@ -87,11 +125,21 @@ def _pick_best_agent(
for agent, data in agents_data.items():
if not _findings_eligible(data):
continue
+ persisted = persisted_agents.get(agent) if isinstance(persisted_agents, dict) else None
+ overall = score_value(persisted.get("overall_with_skill")) if isinstance(persisted, dict) else None
+ if overall is not None:
+ if overall > best_score:
+ best_score = overall
+ best_agent = agent
+ continue
with_scores = data.get("with_skill", {})
- if not with_scores:
+ if not isinstance(with_scores, dict) or not with_scores:
continue
metrics = [m for m in DISPLAY_METRICS if m in with_scores] or list(DISPLAY_METRICS)
- overall = sum(with_scores.get(m, 0.0) for m in metrics) / len(metrics)
+ metric_scores = [score for metric in metrics if (score := metric_value(with_scores, metric)) is not None]
+ if len(metric_scores) != len(metrics):
+ continue
+ overall = sum(metric_scores) / len(metric_scores)
if overall > best_score:
best_score = overall
best_agent = agent
@@ -138,31 +186,36 @@ def _agent_model_label(
return f"{agent} / {model}" if model else agent
-def _details_for_findings(reward: dict[str, Any]) -> dict[str, Any]:
- details = reward.get("details")
- out = dict(details) if isinstance(details, dict) else {}
+def _detail_for_finding(reward: dict[str, Any], metric: str) -> Any:
+ """Return one metric's detail without copying an untrusted details map."""
custom_details = reward.get("custom_details")
+ if metric not in DISPLAY_METRICS and isinstance(custom_details, dict) and metric in custom_details:
+ return custom_details[metric]
+ details = reward.get("details")
+ if isinstance(details, dict) and metric in details:
+ return details[metric]
if isinstance(custom_details, dict):
- for metric, detail in custom_details.items():
- out.setdefault(str(metric), detail)
- return out
+ return custom_details.get(metric)
+ return None
def _finding_metric_names(rewards: list[dict[str, Any]]) -> list[str]:
custom_names: set[str] = set()
for reward in rewards:
- custom_names.update(extract_custom_metrics(reward))
- custom_details = reward.get("custom_details")
- if isinstance(custom_details, dict):
- custom_names.update(str(metric) for metric in custom_details)
+ for name in extract_custom_metrics(reward):
+ custom_names.add(name)
+ if len(custom_names) >= MAX_CUSTOM_METRICS:
+ break
+ if len(custom_names) >= MAX_CUSTOM_METRICS:
+ break
return list(DISPLAY_METRICS) + sorted(custom_names.difference(DISPLAY_METRICS))
def _metric_score(reward: dict[str, Any], metric: str) -> float | None:
- value = reward.get(metric)
- if isinstance(value, int | float) and not isinstance(value, bool):
- numeric = float(value)
- return numeric if math.isfinite(numeric) else None
+ if metric in DISPLAY_METRICS:
+ if metric not in metric_set_for_reward(reward)[1]:
+ return None
+ return metric_value(reward, metric)
return extract_custom_metrics(reward).get(metric)
@@ -192,12 +245,13 @@ def _extract_findings(
metric_values = [score for reward in reward_group if (score := _metric_score(reward, metric)) is not None]
trial_details = []
for reward in reward_group:
- details = _details_for_findings(reward)
- if metric in details:
+ score = _metric_score(reward, metric)
+ detail = _detail_for_finding(reward, metric)
+ if detail is not None and (metric not in DISPLAY_METRICS or score is not None):
trial_details.append(
{
- "score": _metric_score(reward, metric),
- "detail": details[metric],
+ "score": score,
+ "detail": detail,
"entry_id": reward.get("entry_id", "?"),
}
)
@@ -225,11 +279,18 @@ def _extract_findings(
for t in trials:
d = t["detail"]
if isinstance(d, dict):
- metric_refs.extend(d.get("evidence_refs") or [])
+ raw_refs = d.get("evidence_refs")
+ if isinstance(raw_refs, list):
+ metric_refs.extend(raw_refs)
+ elif isinstance(raw_refs, dict | str):
+ metric_refs.append(raw_refs)
_seen: set[tuple[Any, ...]] = set()
_refs: list[dict[str, Any]] = []
- for r in metric_refs:
- k = (r.get("source"), r.get("json_pointer"), r.get("kind"), r.get("path"))
+ for raw_ref in metric_refs:
+ if not isinstance(raw_ref, dict | str):
+ continue
+ r = _resolve_evidence_ref(raw_ref, {})
+ k = tuple(str(r.get(field) or "") for field in ("source", "json_pointer", "kind", "path"))
if k not in _seen:
_seen.add(k)
_refs.append(r)
@@ -243,7 +304,7 @@ def _extract_findings(
"severity": "ok",
"score": avg_score,
"reasons": reasons[:2],
- "evidence_refs": _refs[:8],
+ "evidence_refs": _refs[:_MAX_FINDING_EVIDENCE_REFS],
}
)
else:
@@ -256,7 +317,7 @@ def _extract_findings(
"severity": severity,
"score": avg_score,
"reasons": reasons[:4],
- "evidence_refs": _refs[:8],
+ "evidence_refs": _refs[:_MAX_FINDING_EVIDENCE_REFS],
}
)
@@ -307,63 +368,79 @@ def _collect_fail_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
for trial in trials:
detail = trial["detail"]
+ if not isinstance(detail, dict):
+ continue
if metric == "behavior_check":
- for r in detail.get("results", []):
- if not r.get("passed") and r.get("reason"):
- reasons.append(r["reason"])
+ results = detail.get("results")
+ for result in results if isinstance(results, list) else []:
+ if not isinstance(result, dict) or result.get("passed") is not False:
+ continue
+ if reason := _bounded_reason_text(result.get("reason")):
+ reasons.append(reason)
elif metric == "accuracy":
- criteria = detail.get("criteria", {})
+ criteria = detail.get("criteria")
+ criteria = criteria if isinstance(criteria, dict) else {}
for crit, passed in criteria.items():
if not passed:
- reasons.append(f"{crit} failed")
- if detail.get("reason"):
- reasons.append(detail["reason"])
+ reasons.append(f"{str(crit)[:480]} failed")
+ if reason := _bounded_reason_text(detail.get("reason")):
+ reasons.append(reason)
elif metric in ("goal_accuracy", "security"):
- if detail.get("reason"):
- reasons.append(detail["reason"])
- for finding in detail.get("findings", []):
+ if reason := _bounded_reason_text(detail.get("reason")):
+ reasons.append(reason)
+ findings = detail.get("findings")
+ for finding in findings if isinstance(findings, list) else []:
if isinstance(finding, str):
- reasons.append(finding)
+ if reason := _bounded_reason_text(finding):
+ reasons.append(reason)
elif isinstance(finding, dict):
- message = str(finding.get("message") or finding.get("type") or "")
- attribution = str(finding.get("attribution") or "")
- explanation = str(finding.get("attribution_explanation") or "")
+ message = _bounded_reason_text(finding.get("message") or finding.get("type"))
+ attribution = _bounded_reason_text(finding.get("attribution"))
+ explanation = _bounded_reason_text(finding.get("attribution_explanation"))
if attribution:
message = f"{message} | Attribution: {attribution.replace('_', ' ')}"
if explanation:
message = f"{message}. {explanation}"
if message:
reasons.append(message)
+ if completeness := _bounded_reason_text(detail.get("attribution_completeness")):
+ reasons.append(completeness)
elif metric == "skill_execution":
for check_name, check_data in detail.items():
if isinstance(check_data, dict) and not check_data.get("passed", True):
- reasons.append(check_data.get("reason", f"{check_name} failed"))
+ reason = _bounded_reason_text(check_data.get("reason")) or f"{str(check_name)[:480]} failed"
+ reasons.append(reason)
if isinstance(check_data, dict) and check_name == "error_recovery":
- for corr in check_data.get("corrections", []):
- fault = corr.get("fault", "unknown")
- err = corr.get("error", "")[:100]
+ corrections = check_data.get("corrections")
+ for corr in corrections if isinstance(corrections, list) else []:
+ if not isinstance(corr, dict):
+ continue
+ fault = _bounded_reason_text(corr.get("fault")) or "unknown"
+ err = _bounded_reason_text(corr.get("error"), max_len=100)
reasons.append(f"[{fault} fault] {err}")
elif metric == "skill_efficiency":
for check_name, check_data in detail.items():
if isinstance(check_data, dict) and not check_data.get("passed", True):
- reasons.append(check_data.get("reason", f"{check_name} failed"))
+ reason = _bounded_reason_text(check_data.get("reason")) or f"{str(check_name)[:480]} failed"
+ reasons.append(reason)
else:
- if isinstance(detail, dict) and detail.get("reason"):
- reasons.append(str(detail["reason"]))
- if isinstance(detail, dict):
- for finding in detail.get("findings", []):
- if isinstance(finding, str):
- reasons.append(finding)
- elif isinstance(finding, dict):
- message = str(finding.get("message") or finding.get("reason") or "")
- if message:
- reasons.append(message)
+ if reason := _bounded_reason_text(detail.get("reason")):
+ reasons.append(reason)
+ findings = detail.get("findings")
+ for finding in findings if isinstance(findings, list) else []:
+ if isinstance(finding, str):
+ if reason := _bounded_reason_text(finding):
+ reasons.append(reason)
+ elif isinstance(finding, dict) and (
+ reason := _bounded_reason_text(finding.get("message") or finding.get("reason"))
+ ):
+ reasons.append(reason)
seen: set[str] = set()
deduped: list[str] = []
@@ -384,44 +461,55 @@ def _collect_pass_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
continue
if metric == "behavior_check":
- passed_count = sum(1 for r in detail.get("results", []) if r.get("passed"))
- total = len(detail.get("results", []))
+ raw_results = detail.get("results")
+ results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
+ passed_count = sum(1 for result in results if result.get("passed") is True)
+ total = len(results)
if total:
reasons.append(f"{passed_count}/{total} expected behaviors observed")
- if detail.get("reason"):
- reasons.append(detail["reason"])
+ if reason := _bounded_reason_text(detail.get("reason")):
+ reasons.append(reason)
elif metric == "accuracy":
- criteria = detail.get("criteria", {})
+ criteria = detail.get("criteria")
+ criteria = criteria if isinstance(criteria, dict) else {}
passed = [k for k, v in criteria.items() if v]
if passed:
- reasons.append(f"Passed: {', '.join(passed)}")
- if detail.get("reason"):
- reasons.append(detail["reason"])
+ reasons.append(f"Passed: {', '.join(str(item) for item in passed)[:504]}")
+ if reason := _bounded_reason_text(detail.get("reason")):
+ reasons.append(reason)
elif metric in ("goal_accuracy", "security"):
- if detail.get("reason"):
- reasons.append(detail["reason"])
- for finding in detail.get("findings", []):
+ if reason := _bounded_reason_text(detail.get("reason")):
+ reasons.append(reason)
+ raw_findings = detail.get("findings")
+ for finding in raw_findings if isinstance(raw_findings, list) else []:
if not isinstance(finding, dict):
continue
if finding.get("score_impact"):
continue
- message = str(finding.get("message") or "")
- attribution = str(finding.get("attribution") or "")
+ message = _bounded_reason_text(finding.get("message"))
+ attribution = _bounded_reason_text(finding.get("attribution"))
if attribution:
message = f"{message} | Attribution: {attribution.replace('_', ' ')}"
if message:
reasons.append(message)
- if detail.get("end_state"):
- reasons.append(detail["end_state"])
+ if completeness := _bounded_reason_text(detail.get("attribution_completeness")):
+ reasons.append(completeness)
+ if end_state := _bounded_reason_text(detail.get("end_state")):
+ reasons.append(end_state)
else:
- if detail.get("reason"):
- reasons.append(str(detail["reason"]))
+ if reason := _bounded_reason_text(detail.get("reason")):
+ reasons.append(reason)
for check_data in detail.values():
- if isinstance(check_data, dict) and check_data.get("passed") and check_data.get("reason"):
- reasons.append(check_data["reason"])
+ if (
+ isinstance(check_data, dict)
+ and check_data.get("passed")
+ and check_data.get("reason")
+ and (reason := _bounded_reason_text(check_data.get("reason")))
+ ):
+ reasons.append(reason)
seen: set[str] = set()
deduped: list[str] = []
@@ -433,6 +521,15 @@ def _collect_pass_reasons(metric: str, trials: list[dict[str, Any]]) -> list[str
return deduped
+def _bounded_reason_text(value: Any, *, max_len: int = 512) -> str:
+ """Normalize scalar evaluator explanations without expanding malformed containers."""
+ if isinstance(value, str):
+ return value[:max_len]
+ if isinstance(value, int | float | bool):
+ return str(value)[:max_len]
+ return ""
+
+
def _build_evidence_ref_lookup(rewards: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""Build a lookup from compact string key ``source#json_pointer`` to full dict ref.
@@ -463,6 +560,16 @@ def _build_evidence_ref_lookup(rewards: list[dict[str, Any]]) -> dict[str, dict[
return lookup
+def _bounded_evidence_ref(ref: dict[str, Any]) -> dict[str, str]:
+ """Project one evidence reference into the bounded report schema."""
+ bounded: dict[str, str] = {}
+ for field, limit in _EVIDENCE_REF_TEXT_LIMITS.items():
+ value = ref.get(field)
+ if value not in (None, ""):
+ bounded[field] = _bounded_json_text(value, max_encoded_bytes=limit)
+ return bounded
+
+
def _resolve_evidence_ref(ref: Any, lookup: dict[str, dict[str, Any]]) -> dict[str, Any]:
"""Resolve a single evidence ref to a dict.
@@ -472,16 +579,16 @@ def _resolve_evidence_ref(ref: Any, lookup: dict[str, dict[str, Any]]) -> dict[s
the string, with ``kind`` set to ``"evidence"``.
"""
if isinstance(ref, dict):
- return ref
- ref_str = str(ref)
+ return _bounded_evidence_ref(ref)
+ ref_str = str(ref)[:1024]
if ref_str in lookup:
- return lookup[ref_str]
+ return _bounded_evidence_ref(lookup[ref_str])
# Parse the compact string into a minimal dict
if "#" in ref_str:
source, _, pointer = ref_str.partition("#")
else:
source, pointer = ref_str, ""
- return {"source": source, "json_pointer": pointer, "kind": "evidence"}
+ return _bounded_evidence_ref({"source": source, "json_pointer": pointer, "kind": "evidence"})
def _generate_suggestions_structured(
@@ -517,13 +624,24 @@ def _generate_suggestions_structured(
error_recovery_info = []
for reward in rewards:
details = reward.get("details", {})
- bc = details.get("behavior_check", {})
- for r in bc.get("results", []):
- if not r.get("passed"):
- behavior_failures.append(r.get("reason", ""))
- er = details.get("skill_execution", {}).get("error_recovery", {})
- for corr in er.get("corrections", []):
- error_recovery_info.append(f"[{corr.get('fault', '?')}] {corr.get('error', '')[:150]}")
+ if not isinstance(details, dict):
+ continue
+ behavior = details.get("behavior_check")
+ behavior_results = behavior.get("results") if isinstance(behavior, dict) else None
+ for result in behavior_results if isinstance(behavior_results, list) else []:
+ if not isinstance(result, dict) or result.get("passed") is not False:
+ continue
+ if reason := _bounded_reason_text(result.get("reason")):
+ behavior_failures.append(reason)
+ skill_execution = details.get("skill_execution")
+ error_recovery = skill_execution.get("error_recovery") if isinstance(skill_execution, dict) else None
+ corrections = error_recovery.get("corrections") if isinstance(error_recovery, dict) else None
+ for correction in corrections if isinstance(corrections, list) else []:
+ if not isinstance(correction, dict):
+ continue
+ fault = _bounded_reason_text(correction.get("fault")) or "?"
+ error = _bounded_reason_text(correction.get("error"), max_len=150)
+ error_recovery_info.append(f"[{fault}] {error}")
evidence_lines = []
for f in failed_findings:
@@ -576,21 +694,34 @@ def _generate_suggestions_structured(
result: list[dict[str, Any]] = []
for item in parsed[:4]:
if isinstance(item, dict):
- raw_refs = list(item.get("evidence_refs") or [])
+ raw_refs = list(item.get("evidence_refs") or [])[:_MAX_FINDING_EVIDENCE_REFS]
resolved_refs = [_resolve_evidence_ref(r, ref_lookup) for r in raw_refs]
result.append(
{
- "suggestion": str(item.get("suggestion", "")),
- "dimension": str(item.get("dimension", "")),
+ "suggestion": str(item.get("suggestion", ""))[:_MAX_FINDING_TEXT_CHARS],
+ "dimension": str(item.get("dimension", ""))[:256],
"evidence_refs": resolved_refs,
}
)
elif isinstance(item, str):
- result.append({"suggestion": item, "dimension": "", "evidence_refs": []})
+ result.append(
+ {
+ "suggestion": item[:_MAX_FINDING_TEXT_CHARS],
+ "dimension": "",
+ "evidence_refs": [],
+ }
+ )
if result:
return result
if isinstance(parsed, dict) and "suggestions" in parsed:
- return [{"suggestion": str(s), "dimension": "", "evidence_refs": []} for s in parsed["suggestions"][:4]]
+ return [
+ {
+ "suggestion": str(s)[:_MAX_FINDING_TEXT_CHARS],
+ "dimension": "",
+ "evidence_refs": [],
+ }
+ for s in parsed["suggestions"][:_MAX_SUGGESTIONS]
+ ]
return [{"suggestion": s, "dimension": "", "evidence_refs": []} for s in _fallback_suggestions(findings)]
except Exception as e:
@@ -665,7 +796,7 @@ def _passing_skill_suggestions(
) -> list[str]:
"""Generate improvement suggestions even when all metrics pass."""
suggestions: list[str] = []
- num_trials = len(rewards)
+ num_trials = len(report_data.logical_trial_reward_groups(rewards))
lowest = min(findings, key=lambda f: f["score"]) if findings else None
if lowest and lowest["score"] < 0.95:
@@ -708,10 +839,10 @@ def _harbor_viewer_evidence_links(rewards: list[dict[str, Any]]) -> list[str]:
if isinstance(evidence_urls, list):
for item in evidence_urls:
if isinstance(item, dict) and item.get("url"):
- links.append(str(item["url"]))
+ links.append(str(item["url"])[:_MAX_FINDING_TEXT_CHARS])
trial_url = harbor_viewer.get("trial_url")
if trial_url:
- links.append(str(trial_url))
+ links.append(str(trial_url)[:_MAX_FINDING_TEXT_CHARS])
seen: set[str] = set()
deduped: list[str] = []
@@ -731,8 +862,8 @@ def _prioritized_evidence_rewards(rewards: list[dict[str, Any]]) -> list[dict[st
def _reward_has_failing_signal(reward: dict[str, Any]) -> bool:
for metric in DISPLAY_METRICS:
- value = reward.get(metric)
- if isinstance(value, int | float) and not isinstance(value, bool) and float(value) < 0.8:
+ value = _metric_score(reward, metric)
+ if value is not None and value < 0.8:
return True
details = reward.get("details")
if not isinstance(details, dict):
@@ -741,7 +872,8 @@ def _reward_has_failing_signal(reward: dict[str, Any]) -> bool:
if not isinstance(detail, dict):
continue
score = detail.get("score")
- if isinstance(score, int | float) and not isinstance(score, bool) and float(score) < 0.8:
+ numeric_score = extract_custom_metrics({"detail_score": score}).get("detail_score")
+ if numeric_score is not None and numeric_score < 0.8:
return True
results = detail.get("results")
if isinstance(results, list) and any(
@@ -767,10 +899,86 @@ def add_evidence_links_to_suggestions(
linked.append(text)
continue
punctuation = "" if text.endswith((".", "!", "?")) else "."
- linked.append(f"{text}{punctuation} Evidence: {links[min(index, len(links) - 1)]}")
+ linked.append(f"{text}{punctuation} Evidence: {links[min(index, len(links) - 1)]}"[:_MAX_FINDING_TEXT_CHARS])
return linked
+def _bounded_findings_payload(
+ *,
+ skill_name: str,
+ agent: str,
+ findings: list[dict[str, Any]],
+ suggestions: list[str],
+ suggestion_mode: str,
+ suggestions_v2: list[dict[str, Any]] | None,
+) -> dict[str, Any]:
+ """Project a findings artifact into a fixed, browser-loadable schema."""
+ bounded_findings: list[dict[str, Any]] = []
+ for finding in findings[:_MAX_FINDINGS]:
+ if not isinstance(finding, dict):
+ continue
+ score = finding.get("score")
+ try:
+ score_is_finite = (
+ isinstance(score, int | float) and not isinstance(score, bool) and math.isfinite(float(score))
+ )
+ except (OverflowError, ValueError):
+ score_is_finite = False
+ if not score_is_finite:
+ continue
+ raw_reasons = finding.get("reasons")
+ raw_refs = finding.get("evidence_refs")
+ bounded_findings.append(
+ {
+ "metric": _bounded_json_text(finding.get("metric"), max_encoded_bytes=256),
+ "label": _bounded_json_text(finding.get("label"), max_encoded_bytes=512),
+ "severity": _bounded_json_text(finding.get("severity"), max_encoded_bytes=64),
+ "score": float(score),
+ "reasons": [
+ _bounded_json_text(reason, max_encoded_bytes=512)
+ for reason in (raw_reasons if isinstance(raw_reasons, list) else [])[:_MAX_FINDING_REASONS]
+ ],
+ "evidence_refs": [
+ _bounded_evidence_ref(ref)
+ for ref in (raw_refs if isinstance(raw_refs, list) else [])[:_MAX_FINDING_EVIDENCE_REFS]
+ if isinstance(ref, dict)
+ ],
+ }
+ )
+
+ bounded_v2: list[dict[str, Any]] = []
+ for suggestion in (suggestions_v2 or [])[:_MAX_SUGGESTIONS]:
+ if not isinstance(suggestion, dict):
+ continue
+ raw_refs = suggestion.get("evidence_refs")
+ bounded = {
+ "suggestion": _bounded_json_text(suggestion.get("suggestion"), max_encoded_bytes=4096),
+ "dimension": _bounded_json_text(suggestion.get("dimension"), max_encoded_bytes=256),
+ "evidence_refs": [
+ _bounded_evidence_ref(ref)
+ for ref in (raw_refs if isinstance(raw_refs, list) else [])[:_MAX_FINDING_EVIDENCE_REFS]
+ if isinstance(ref, dict)
+ ],
+ }
+ if suggestion.get("trial_id") not in (None, ""):
+ bounded["trial_id"] = _bounded_json_text(suggestion["trial_id"], max_encoded_bytes=512)
+ bounded_v2.append(bounded)
+
+ return {
+ "skill_name": _bounded_json_text(skill_name, max_encoded_bytes=512),
+ "agent": _bounded_json_text(agent, max_encoded_bytes=512),
+ "suggestion_mode": _bounded_json_text(suggestion_mode, max_encoded_bytes=64),
+ "findings": bounded_findings,
+ "findings_total": len(findings),
+ "findings_shown": len(bounded_findings),
+ "findings_truncated": len(bounded_findings) < len(findings),
+ "suggestions": [
+ _bounded_json_text(suggestion, max_encoded_bytes=4096) for suggestion in suggestions[:_MAX_SUGGESTIONS]
+ ],
+ "suggestions_v2": bounded_v2,
+ }
+
+
def _write_findings_artifact(
*,
results_dir: Path,
@@ -784,18 +992,42 @@ def _write_findings_artifact(
artifact = _findings_artifact_path(results_dir, agent)
if artifact is None:
return None
- payload = {
- "skill_name": skill_name,
- "agent": agent,
- "suggestion_mode": suggestion_mode,
- "findings": findings,
- "suggestions": suggestions,
- "suggestions_v2": suggestions_v2 or [],
- }
+ payload = _bounded_findings_payload(
+ skill_name=skill_name,
+ agent=agent,
+ findings=findings,
+ suggestions=suggestions,
+ suggestion_mode=suggestion_mode,
+ suggestions_v2=suggestions_v2,
+ )
try:
artifact.parent.mkdir(parents=True, exist_ok=True)
- artifact.write_text(json.dumps(redact_sensitive_data(payload), indent=2), encoding="utf-8")
- except OSError as e:
+ safe_payload = redact_sensitive_data(payload, max_str_len=_MAX_FINDING_TEXT_CHARS)
+ encoded = json.dumps(
+ safe_payload,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ).encode("utf-8")
+ if len(encoded) > report_data._MAX_JSON_BYTES:
+ for finding in safe_payload.get("findings", []):
+ if isinstance(finding, dict):
+ finding["evidence_refs"] = []
+ for suggestion in safe_payload.get("suggestions_v2", []):
+ if isinstance(suggestion, dict):
+ suggestion["evidence_refs"] = []
+ safe_payload["evidence_refs_omitted"] = True
+ safe_payload["evidence_refs_omitted_reason"] = "findings artifact size limit"
+ encoded = json.dumps(
+ safe_payload,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ ).encode("utf-8")
+ if len(encoded) > report_data._MAX_JSON_BYTES:
+ raise ValueError("bounded findings artifact exceeds report loader limit")
+ write_output_file_atomically(artifact, encoded)
+ except (OSError, TypeError, ValueError) as e:
logger.debug("Failed to write findings artifact %s: %s", artifact, e)
return None
return artifact
@@ -823,8 +1055,9 @@ def display_findings_report(
for agent in report_agents:
_remove_stale_findings_artifact(results_dir, agent)
+ loaded_agents = report_data.load_agent_data(results_dir)
if len(harbor_agents) > 1:
- best_agent = _pick_best_agent(agents_data)
+ best_agent = _pick_best_agent(agents_data, loaded_agents)
if best_agent:
best_agent_label = _agent_model_label(best_agent, harbor_result, agents_data)
console.print(
@@ -843,7 +1076,6 @@ def display_findings_report(
):
return set()
- loaded_agents = report_data.load_agent_data(results_dir)
agent_reports: dict[str, tuple[list[dict[str, Any]], list[dict[str, Any]]]] = {}
for agent in report_agents:
agent_data = agents_data.get(agent)
diff --git a/src/skillevaluator/tier3/harbor/report_data.py b/src/skillevaluator/tier3/harbor/report_data.py
index 31316322..b81b93c6 100644
--- a/src/skillevaluator/tier3/harbor/report_data.py
+++ b/src/skillevaluator/tier3/harbor/report_data.py
@@ -13,6 +13,7 @@
import heapq
import json
import logging
+import math
import os
import stat
from collections.abc import Callable, Iterable
@@ -20,7 +21,7 @@
from pathlib import Path
from typing import Any
-from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, LEGACY_METRICS
+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, LEGACY_METRICS, metric_set_for_reward
logger = logging.getLogger(__name__)
@@ -28,6 +29,7 @@
_MAX_JSON_DEPTH = 64
_MAX_JSON_NODES = 50_000
_MAX_JSON_NUMBER_CHARS = 4_300
+_MAX_JSON_SAFE_INTEGER = (1 << 53) - 1
_MAX_AGENTS = 64
_MAX_AGENT_PATHS_SCANNED = 512
_MAX_TRIALS_PER_CONDITION = 512
@@ -38,16 +40,33 @@
_MAX_DIAGNOSTIC_REASONS = 8
_INVALID_JSON = object()
+DATASET_SNAPSHOT_MAX_BYTES = _MAX_JSON_BYTES
+DATASET_SNAPSHOT_MAX_DEPTH = _MAX_JSON_DEPTH
+DATASET_SNAPSHOT_MAX_NODES = _MAX_JSON_NODES
+DATASET_SNAPSHOT_LIMIT_ERROR = (
+ "Dataset snapshot exceeds the 2 MiB, depth-64, or 50,000-node publication limit; "
+ "reduce dataset size or structural complexity."
+)
+
__all__ = (
"DATASET_SNAPSHOT_DIGEST_ALGORITHM",
+ "DATASET_SNAPSHOT_LIMIT_ERROR",
+ "DATASET_SNAPSHOT_MAX_BYTES",
+ "DATASET_SNAPSHOT_MAX_DEPTH",
+ "DATASET_SNAPSHOT_MAX_NODES",
+ "DatasetSnapshotContractError",
+ "aggregate_execution_error_details",
"build_dataset_snapshot",
+ "dataset_snapshot_manifest",
"deduplicate_dataset_entries",
+ "encode_dataset_snapshot",
"load_agent_data",
"load_dataset",
"load_dataset_snapshot",
"load_staged_harbor_dataset",
"logical_trial_reward_groups",
"metrics_for_agents",
+ "metrics_for_condition",
"summarize_dataset_entries",
)
@@ -55,6 +74,13 @@
DATASET_SNAPSHOT_SCHEMA_VERSION = "1.0"
+class DatasetSnapshotContractError(ValueError):
+ """Raised when exact dataset truth cannot fit the public artifact contract."""
+
+ def __init__(self) -> None:
+ super().__init__(DATASET_SNAPSHOT_LIMIT_ERROR)
+
+
def _canonical_dataset_json(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
@@ -110,6 +136,53 @@ def build_dataset_snapshot(entries: list[dict[str, Any]], *, evaluator_version:
}
+def encode_dataset_snapshot(snapshot: object) -> bytes:
+ """Encode one snapshot within the same bounds enforced by the report loader."""
+ try:
+ _validate_json_tree(snapshot)
+ stack = [snapshot]
+ while stack:
+ current = stack.pop()
+ if isinstance(current, dict):
+ stack.extend(current.values())
+ elif isinstance(current, list):
+ stack.extend(current)
+ elif isinstance(current, bool) or current is None or isinstance(current, str):
+ continue
+ elif isinstance(current, int):
+ if abs(current) > _MAX_JSON_SAFE_INTEGER:
+ raise ValueError("browser-unsafe JSON integer")
+ elif isinstance(current, float):
+ if not math.isfinite(current):
+ raise ValueError("non-finite JSON number")
+ else:
+ raise TypeError("value is not JSON serializable")
+ encoded = json.dumps(
+ snapshot,
+ separators=(",", ":"),
+ ensure_ascii=True,
+ allow_nan=False,
+ sort_keys=True,
+ ).encode("utf-8")
+ except (MemoryError, RecursionError, TypeError, UnicodeError, ValueError):
+ raise DatasetSnapshotContractError from None
+ if len(encoded) > DATASET_SNAPSHOT_MAX_BYTES:
+ raise DatasetSnapshotContractError
+ return encoded
+
+
+def dataset_snapshot_manifest(snapshot: dict[str, Any]) -> dict[str, Any]:
+ """Return validated dataset metadata suitable for embedding in ``result.json``."""
+ encode_dataset_snapshot(snapshot)
+ return {
+ "schema_version": snapshot.get("schema_version"),
+ "evaluator_version": snapshot.get("evaluator_version"),
+ "dataset_summary": dict(snapshot.get("dataset_summary", {})),
+ "dataset_digest": snapshot.get("dataset_digest"),
+ "dataset_digest_algorithm": snapshot.get("dataset_digest_algorithm"),
+ }
+
+
def load_dataset_snapshot(run_dir: Path) -> dict[str, Any] | None:
"""Load a validated run-owned dataset snapshot, if one was persisted."""
path = run_dir / "dataset_snapshot.json"
@@ -117,6 +190,10 @@ def load_dataset_snapshot(run_dir: Path) -> dict[str, Any] | None:
snapshot = _load_bounded_json(path, diagnostics, artifact="dataset_snapshot")
if not isinstance(snapshot, dict) or snapshot.get("schema_version") != DATASET_SNAPSHOT_SCHEMA_VERSION:
return None
+ try:
+ encode_dataset_snapshot(snapshot)
+ except DatasetSnapshotContractError:
+ return None
dataset = snapshot.get("dataset")
summary = snapshot.get("dataset_summary")
version = snapshot.get("evaluator_version")
@@ -414,21 +491,30 @@ def _load_bounded_jsonl(raw: bytes, diagnostics: list[dict[str, Any]]) -> list[A
def _metrics_for_rewards(rewards: list[dict[str, Any]]) -> list[str]:
- if any(isinstance(reward.get("security"), int | float) for reward in rewards):
+ inferred = [metric_set_for_reward(reward)[1] for reward in rewards]
+ if any(metrics == DEFAULT_METRICS for metrics in inferred):
return list(DEFAULT_METRICS)
- if any(any(isinstance(reward.get(metric), int | float) for metric in LEGACY_METRICS) for reward in rewards):
+ if any(metrics == LEGACY_METRICS for metrics in inferred):
return list(LEGACY_METRICS)
return []
-def _skill_evaluator_metrics_for_agent(agent_info: dict[str, Any]) -> list[str]:
- configured = agent_info.get("metrics_with_skill")
+def metrics_for_condition(agent_info: dict[str, Any], condition: str) -> list[str]:
+ """Return the standard metric contract declared by one agent condition."""
+ if condition == "with_skill":
+ metrics_key, scores_key, rewards_key = "metrics_with_skill", "with_skill", "rewards"
+ elif condition == "without_skill":
+ metrics_key, scores_key, rewards_key = "metrics_without_skill", "without_skill", "rewards_baseline"
+ else:
+ return []
+
+ configured = agent_info.get(metrics_key)
if isinstance(configured, list):
return [str(metric) for metric in configured]
- scores = agent_info.get("with_skill", {})
+ scores = agent_info.get(scores_key, {})
if isinstance(scores, dict) and "security" in scores:
return list(DEFAULT_METRICS)
- rewards = agent_info.get("rewards", [])
+ rewards = agent_info.get(rewards_key, [])
return _metrics_for_rewards(rewards) if isinstance(rewards, list) else []
@@ -436,7 +522,7 @@ def metrics_for_agents(agents: dict[str, dict[str, Any]]) -> list[str]:
"""Return the canonical default or legacy metric set represented by agents."""
saw_metrics = False
for info in agents.values():
- metrics = _skill_evaluator_metrics_for_agent(info)
+ metrics = metrics_for_condition(info, "with_skill")
if metrics:
saw_metrics = True
if "security" in metrics:
@@ -464,6 +550,63 @@ def _nonnegative_counter(value: Any) -> int:
return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0
+def _execution_error_details(data: dict[str, Any], displayed_count: int) -> dict[str, Any]:
+ """Normalize bounded execution-error detail metadata around visible errors."""
+ raw_total = data.get("execution_error_details_total")
+ declared_total = (
+ raw_total
+ if isinstance(raw_total, int) and not isinstance(raw_total, bool) and 0 <= raw_total <= _MAX_JSON_SAFE_INTEGER
+ else displayed_count
+ )
+ raw_shown = data.get("execution_error_details_shown")
+ declared_shown = (
+ raw_shown
+ if isinstance(raw_shown, int) and not isinstance(raw_shown, bool) and 0 <= raw_shown <= _MAX_JSON_SAFE_INTEGER
+ else displayed_count
+ )
+ declared_total = max(declared_total, declared_shown, displayed_count)
+ truncated = data.get("execution_error_details_truncated") is True
+ if truncated and declared_total <= displayed_count:
+ declared_total = min(_MAX_JSON_SAFE_INTEGER, displayed_count + 1)
+ return {
+ "execution_error_details_total": declared_total,
+ "execution_error_details_shown": displayed_count,
+ "execution_error_details_truncated": truncated or displayed_count < declared_total,
+ }
+
+
+def aggregate_execution_error_details(
+ summaries: Iterable[dict[str, Any]],
+ displayed_count: int,
+) -> dict[str, Any]:
+ """Sum condition occurrence counts while deduplicating only displayed text."""
+ total = 0
+ child_truncated = False
+ for summary in summaries:
+ raw_errors = summary.get("execution_errors")
+ visible_count = len([error for error in raw_errors if error]) if isinstance(raw_errors, list) else 0
+ details = _execution_error_details(summary, visible_count)
+ total = min(_MAX_JSON_SAFE_INTEGER, total + details["execution_error_details_total"])
+ child_truncated = child_truncated or details["execution_error_details_truncated"]
+ total = max(total, displayed_count)
+ if child_truncated and total <= displayed_count:
+ total = min(_MAX_JSON_SAFE_INTEGER, displayed_count + 1)
+ return {
+ "execution_error_details_total": total,
+ "execution_error_details_shown": displayed_count,
+ "execution_error_details_truncated": child_truncated or displayed_count < total,
+ }
+
+
+def _trajectory_token_counter(value: Any) -> int | None:
+ """Return a browser-safe token counter without inventing zero usage."""
+ return (
+ value
+ if isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= _MAX_JSON_SAFE_INTEGER
+ else None
+ )
+
+
def _condition_status(agent_info: dict[str, Any], condition: str) -> str:
conditions = agent_info.get("conditions")
data = conditions.get(condition) if isinstance(conditions, dict) else None
@@ -525,10 +668,18 @@ def load_agent_data(
agent_info[metric_key] = data.get("metrics", [])
custom_key = "custom_with_skill" if variant == "with-skill" else "custom_without_skill"
if "custom_scores" in data:
- agent_info[custom_key] = data.get("custom_scores", {})
+ custom_scores = data.get("custom_scores")
+ agent_info[custom_key] = custom_scores if isinstance(custom_scores, dict) else {}
overall_key = "overall_with_skill" if variant == "with-skill" else "overall_without_skill"
if "overall_score" in data:
agent_info[overall_key] = data.get("overall_score")
+ mixed_contract_key = (
+ "mixed_metric_contracts_with_skill"
+ if variant == "with-skill"
+ else "mixed_metric_contracts_without_skill"
+ )
+ if isinstance(data.get("mixed_metric_contracts"), bool):
+ agent_info[mixed_contract_key] = data["mixed_metric_contracts"]
dimension_key = "dimensions_with_skill" if variant == "with-skill" else "dimensions_without_skill"
if "dimensions" in data:
agent_info[dimension_key] = data.get("dimensions", {})
@@ -557,6 +708,7 @@ def load_agent_data(
condition_execution[key] = {
"execution_status": status,
"execution_errors": condition_errors,
+ **_execution_error_details(data, len(condition_errors)),
"expected_attempts": _nonnegative_counter(data.get("expected_attempts")),
"scored_attempts": _nonnegative_counter(data.get("scored_attempts")),
}
@@ -564,6 +716,14 @@ def load_agent_data(
num_trials = data.get("num_trials")
if isinstance(num_trials, int) and not isinstance(num_trials, bool) and num_trials >= 0:
agent_info[count_key] = num_trials
+ reward_row_count_key = "num_reward_rows" if variant == "with-skill" else "num_reward_rows_baseline"
+ num_reward_rows = data.get("num_reward_rows")
+ if (
+ isinstance(num_reward_rows, int)
+ and not isinstance(num_reward_rows, bool)
+ and num_reward_rows >= 0
+ ):
+ agent_info[reward_row_count_key] = num_reward_rows
lift_file = agent_dir / "lift.json"
if lift_file.exists():
@@ -581,13 +741,15 @@ def load_agent_data(
if custom_lift_file.exists():
custom_lift = _load_bounded_json(custom_lift_file, agent_diagnostics, artifact="custom_lift")
if custom_lift is not _INVALID_JSON:
- agent_info["custom_lift"] = custom_lift
+ agent_info["custom_lift"] = custom_lift if isinstance(custom_lift, dict) else {}
for variant_key, variant_dir_name in (("rewards", "with-skill"), ("rewards_baseline", "without-skill")):
trial_list: list[dict[str, Any]] = []
count_key = "num_trials" if variant_key == "rewards" else "num_trials_baseline"
- expected_reward_rows = agent_info.get(count_key)
- rewards_complete = isinstance(expected_reward_rows, int)
+ reward_row_count_key = "num_reward_rows" if variant_key == "rewards" else "num_reward_rows_baseline"
+ expected_logical_trials = agent_info.get(count_key)
+ expected_reward_rows = agent_info.get(reward_row_count_key)
+ rewards_complete = isinstance(expected_logical_trials, int)
trials_dir = agent_dir / variant_dir_name / "trials"
if _is_safe_directory(trials_dir, results_dir):
try:
@@ -638,17 +800,22 @@ def load_agent_data(
final_metrics = trajectory.get("final_metrics", {})
if not isinstance(final_metrics, dict):
final_metrics = {}
- steps = trajectory.get("steps", [])
+ steps = trajectory.get("steps")
reward["_traj"] = {
- "steps": len(steps) if isinstance(steps, list) else 0,
- "prompt_tokens": final_metrics.get("total_prompt_tokens", 0),
- "completion_tokens": final_metrics.get("total_completion_tokens", 0),
- "cached_tokens": final_metrics.get("total_cached_tokens", 0),
+ "steps": len(steps) if isinstance(steps, list) else None,
+ "prompt_tokens": _trajectory_token_counter(final_metrics.get("total_prompt_tokens")),
+ "completion_tokens": _trajectory_token_counter(
+ final_metrics.get("total_completion_tokens")
+ ),
+ "cached_tokens": _trajectory_token_counter(final_metrics.get("total_cached_tokens")),
}
trial_list.append(reward)
else:
- rewards_complete = expected_reward_rows == 0
- if expected_reward_rows != len(trial_list):
+ rewards_complete = expected_logical_trials == 0
+ logical_trial_count = len(logical_trial_reward_groups(trial_list))
+ if expected_logical_trials != logical_trial_count:
+ rewards_complete = False
+ if isinstance(expected_reward_rows, int) and expected_reward_rows != len(trial_list):
rewards_complete = False
agent_info[variant_key] = trial_list
agent_info[f"{variant_key}_complete"] = rewards_complete
@@ -668,11 +835,13 @@ def load_agent_data(
execution_status = "skipped"
else:
execution_status = "succeeded"
+ public_execution_errors = list(dict.fromkeys(execution_errors))
agent_info.update(
{
"conditions": condition_execution,
"execution_status": execution_status,
- "execution_errors": list(dict.fromkeys(execution_errors)),
+ "execution_errors": public_execution_errors,
+ **aggregate_execution_error_details(active_conditions, len(public_execution_errors)),
"expected_attempts": sum(
_nonnegative_counter(condition.get("expected_attempts")) for condition in active_conditions
),
diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py
index c647b390..185ec429 100644
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@ -5,27 +5,34 @@
from __future__ import annotations
+import codecs
+import hashlib
+import importlib.util
import json
import logging
+import math
import os
import re
import shlex
import shutil
+import signal
import stat
import subprocess
+import sys
import tempfile
+import threading
import time
import tomllib
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait
-from contextlib import ExitStack
+from contextlib import ExitStack, suppress
from dataclasses import dataclass, field
from datetime import UTC, datetime
from functools import wraps
from pathlib import Path
from queue import Empty, SimpleQueue
from types import MappingProxyType
-from typing import Any
+from typing import Any, NoReturn
from uuid import uuid4
from skillevaluator import __version__
@@ -36,9 +43,20 @@
_normalize_anthropic_base_url,
resolve_llm_provider,
)
-from skillevaluator.tier3.evals_config import EvalsConfigError, load_evals_config
+from skillevaluator.tier3.case_ids import validate_case_ids
+from skillevaluator.tier3.evals_config import (
+ MAX_HARBOR_TIMEOUT_MULTIPLIER,
+ EvalsConfigError,
+ encode_environment_kwarg,
+ load_evals_config,
+ validate_environment_kwargs,
+)
from skillevaluator.tier3.harbor.adapter import (
+ _RUNTIME_PROCESS_CONTROL_ENV_NAMES,
+ _RUNTIME_PROCESS_CONTROL_ENV_PREFIXES,
+ _VERIFIER_JUDGE_CONTROL_ENV_VARS,
_VERIFIER_JUDGE_MODEL_ENV_VARS,
+ _native_entry_id,
_prevalidate_baseline_skill_candidates,
build_eval_base_image,
find_evals_file,
@@ -50,6 +68,7 @@
)
from skillevaluator.tier3.harbor.artifact_retention import HarborArtifactLifecycle, RetentionOutcome
from skillevaluator.tier3.harbor.collector import (
+ TRUNCATED_AGGREGATE_ATTEMPT_PREFIX,
collect_harbor_results,
harbor_job_passed,
validate_harbor_job_result,
@@ -65,7 +84,10 @@
secret_values_from_environment,
)
from skillevaluator.tier3.harbor.report_data import (
+ DatasetSnapshotContractError,
build_dataset_snapshot,
+ dataset_snapshot_manifest,
+ encode_dataset_snapshot,
load_staged_harbor_dataset,
)
from skillevaluator.tier3.harbor.secure_copy import copytree_secure
@@ -76,6 +98,12 @@
from skillevaluator.tier3.harbor.sensitive_stdin import (
NVIDIA_BUILD_STDIN_SENTINEL as _NVIDIA_BUILD_STDIN_SENTINEL,
)
+from skillevaluator.tier3.harbor.stream_redaction import (
+ MAX_COMMAND_OUTPUT_BYTES,
+ CommandOutputByteBudget,
+ StreamingLogRedactor,
+ StreamingSecretRedactor,
+)
from skillevaluator.tier3.output_provenance import (
mark_generated_output_root,
remove_generated_output_root_if_owned,
@@ -83,37 +111,494 @@
write_output_file_atomically,
)
from skillevaluator.tier3.results_location import publish_latest_results
-from skillevaluator.tier3_environments import DEFAULT_ENV_MODE, ENV_MODE_LOCAL, HARBOR_ENV_MODES
+from skillevaluator.tier3_environments import (
+ DEFAULT_ENV_MODE,
+ ENV_MODE_LOCAL,
+ HARBOR_ENV_MODES,
+ HARBOR_ENVIRONMENT_EXTRAS,
+ HARBOR_V022_ENVIRONMENT_KWARGS,
+)
+from skillevaluator.utils.redaction import is_sensitive_key, redact_sensitive_text
+from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
logger = logging.getLogger(__name__)
+PUBLISHED_EXECUTION_ERRORS_MAX = 256
+PUBLISHED_EXECUTION_ERROR_MAX_CHARS = 4096
+PUBLISHED_EXECUTION_ERROR_MAX_SERIALIZED_BYTES = 4096
+PUBLISHED_EXECUTION_ERRORS_MAX_SERIALIZED_BYTES = 64 * 1024
+_EXECUTION_ERROR_TRUNCATION_MARKER = "..."
+_MAX_JSON_SAFE_INTEGER = (1 << 53) - 1
+FINAL_RESULT_MAX_BYTES = 2 * 1024 * 1024
+FINAL_RESULT_MAX_DEPTH = 64
+FINAL_RESULT_MAX_NODES = 50_000
+_FINAL_RESULT_PROJECTION_SCHEMA_VERSION = "1.0"
+_FINAL_RESULT_CONTRACT_ERROR = (
+ "Final Tier 3 result exceeds the 2 MiB, depth-64, or 50,000-node publication limit after artifact projection"
+)
+
+
+class _FinalResultContractError(ValueError):
+ """Raised before publishing a result the report loader cannot read."""
+
+
+def _reject_nonfinite_json_constant(_constant: str) -> NoReturn:
+ """Reject JSON constants that are outside the interoperable JSON grammar."""
+ raise ValueError("non-finite JSON number")
+
+
+def _validate_final_result_tree(value: object) -> None:
+ """Match the structural envelope enforced by the report JSON loader."""
+ nodes = 0
+ stack: list[tuple[object, int]] = [(value, 1)]
+ while stack:
+ current, depth = stack.pop()
+ nodes += 1
+ if nodes > FINAL_RESULT_MAX_NODES:
+ raise _FinalResultContractError(_FINAL_RESULT_CONTRACT_ERROR)
+ if not isinstance(current, dict | list):
+ continue
+ if depth > FINAL_RESULT_MAX_DEPTH or nodes + len(current) > FINAL_RESULT_MAX_NODES:
+ raise _FinalResultContractError(_FINAL_RESULT_CONTRACT_ERROR)
+ children = current.values() if isinstance(current, dict) else current
+ stack.extend((child, depth + 1) for child in children)
+
+
+def _encode_final_result(value: dict[str, Any]) -> bytes:
+ """Serialize one final result only when the browser/report loader can consume it."""
+ _validate_final_result_tree(value)
+ encoded = _serialize_final_result(value)
+ if len(encoded) > FINAL_RESULT_MAX_BYTES:
+ raise _FinalResultContractError(_FINAL_RESULT_CONTRACT_ERROR)
+ return encoded
+
+
+def _serialize_final_result(value: dict[str, Any]) -> bytes:
+ """Serialize valid JSON without applying the publication size envelope."""
+ try:
+ return json.dumps(value, indent=2, allow_nan=False).encode("utf-8")
+ except (MemoryError, RecursionError, TypeError, UnicodeError, ValueError):
+ raise _FinalResultContractError(_FINAL_RESULT_CONTRACT_ERROR) from None
+
+
+def _result_artifact_reference(run_dir: Path, relative: Path) -> dict[str, Any] | None:
+ """Describe one regular contained artifact by stable relative path and digest."""
+ if relative.is_absolute() or not relative.parts or any(part in {"", ".", ".."} for part in relative.parts):
+ return None
+ try:
+ with SecureRoot(run_dir) as secure_root:
+ raw, _metadata = secure_root.read_bytes(relative, FINAL_RESULT_MAX_BYTES)
+ decoded = json.loads(
+ raw,
+ parse_constant=_reject_nonfinite_json_constant,
+ )
+ if not isinstance(decoded, dict):
+ return None
+ _validate_final_result_tree(decoded)
+ except (OSError, RecursionError, RuntimeError, SecurePathError, UnicodeError, ValueError):
+ return None
+ return {
+ "path": relative.as_posix(),
+ "bytes": len(raw),
+ "sha256": f"sha256:{hashlib.sha256(raw).hexdigest()}",
+ }
+
+
+def _compact_pass_detail(value: object, *, artifact_key: str) -> object:
+ """Retain exact pass aggregates while referencing persisted case details."""
+ if not isinstance(value, dict):
+ return value
+ compact = dict(value)
+ if isinstance(compact.get("cases"), dict):
+ compact["cases"] = {}
+ if isinstance(compact.get("extra_cases"), list):
+ compact["extra_cases"] = []
+ compact["detail_projection"] = {
+ "artifact": artifact_key,
+ "json_pointer": "/pass_at_k",
+ }
+ return compact
+
+
+def _compact_condition_detail(value: object, *, artifact_key: str) -> object:
+ """Retain condition truth while sampling diagnostics stored in its summary."""
+ if not isinstance(value, dict):
+ return value
+ compact = dict(value)
+ raw_errors = compact.get("execution_errors")
+ errors = [str(error) for error in raw_errors if str(error)] if isinstance(raw_errors, list) else []
+ total = compact.get("execution_error_details_total")
+ exact_total = (
+ total if isinstance(total, int) and not isinstance(total, bool) and total >= len(errors) else len(errors)
+ )
+ compact["execution_errors"] = errors[:1]
+ compact["execution_error_details_total"] = exact_total
+ compact["execution_error_details_shown"] = len(compact["execution_errors"])
+ compact["execution_error_details_truncated"] = len(compact["execution_errors"]) < exact_total
+ compact["detail_projection"] = {
+ "artifact": artifact_key,
+ "json_pointer": "",
+ }
+ return compact
+
+
+def _compact_agent_result(
+ agent: str,
+ value: object,
+ *,
+ run_dir: Path,
+) -> tuple[object, dict[str, Any], list[str]]:
+ """Project duplicated agent detail only when its canonical artifact exists."""
+ if not isinstance(value, dict) or Path(agent).name != agent or agent in {"", ".", ".."}:
+ return value, {}, []
+ compact = dict(value)
+ artifact_paths = {
+ "with_skill_summary": Path(agent) / "with-skill" / "summary.json",
+ "without_skill_summary": Path(agent) / "without-skill" / "summary.json",
+ "lift": Path(agent) / "lift.json",
+ "custom_lift": Path(agent) / "custom_lift.json",
+ "pass_at_k_lift": Path(agent) / "pass_at_k_lift.json",
+ "security_attribution": Path(agent) / "security_attribution.json",
+ }
+ references = {
+ key: reference
+ for key, relative in artifact_paths.items()
+ if (reference := _result_artifact_reference(run_dir, relative)) is not None
+ }
+ omitted: list[str] = []
+
+ raw_pass = compact.get("pass_at_k")
+ if isinstance(raw_pass, dict):
+ pass_at_k = dict(raw_pass)
+ for condition, artifact_key in (
+ ("with_skill", "with_skill_summary"),
+ ("without_skill", "without_skill_summary"),
+ ):
+ condition_pass = pass_at_k.get(condition)
+ if artifact_key not in references or not isinstance(condition_pass, dict):
+ continue
+ omitted_pass_fields: list[str] = []
+ if isinstance(condition_pass.get("cases"), dict) and condition_pass["cases"]:
+ omitted_pass_fields.append("cases")
+ if isinstance(condition_pass.get("extra_cases"), list) and condition_pass["extra_cases"]:
+ omitted_pass_fields.append("extra_cases")
+ if omitted_pass_fields:
+ pass_at_k[condition] = _compact_pass_detail(condition_pass, artifact_key=artifact_key)
+ omitted.extend(f"pass_at_k.{condition}.{field}" for field in omitted_pass_fields)
+ compact["pass_at_k"] = pass_at_k
+
+ if "security_attribution" in references and isinstance(compact.get("security_attribution"), dict):
+ security = dict(compact["security_attribution"])
+ if isinstance(security.get("cases"), dict) and security["cases"]:
+ security["cases"] = {}
+ omitted.append("security_attribution.cases")
+ security["detail_projection"] = {
+ "artifact": "security_attribution",
+ "json_pointer": "",
+ }
+ compact["security_attribution"] = security
+
+ conditions = compact.get("conditions")
+ if isinstance(conditions, dict):
+ compact_conditions = dict(conditions)
+ for condition, artifact_key in (
+ ("with_skill", "with_skill_summary"),
+ ("without_skill", "without_skill_summary"),
+ ):
+ condition_detail = compact_conditions.get(condition)
+ if (
+ artifact_key in references
+ and isinstance(condition_detail, dict)
+ and isinstance(condition_detail.get("execution_errors"), list)
+ and condition_detail["execution_errors"]
+ ):
+ compact_conditions[condition] = _compact_condition_detail(
+ condition_detail,
+ artifact_key=artifact_key,
+ )
+ omitted.append(f"conditions.{condition}.execution_errors")
+ compact["conditions"] = compact_conditions
+
+ for failure_field in ("agent_runtime_failures", "trial_failures"):
+ raw_failures = compact.get(failure_field)
+ if not isinstance(raw_failures, dict):
+ continue
+ projected_failures = dict(raw_failures)
+ for condition, artifact_key in (
+ ("with_skill", "with_skill_summary"),
+ ("without_skill", "without_skill_summary"),
+ ):
+ if (
+ artifact_key in references
+ and isinstance(projected_failures.get(condition), list)
+ and projected_failures[condition]
+ ):
+ projected_failures[condition] = []
+ omitted.append(f"{failure_field}.{condition}")
+ compact[failure_field] = projected_failures
+
+ job_failures = compact.get("job_failures")
+ if isinstance(job_failures, dict):
+ projected_job_failures = dict(job_failures)
+ for condition, artifact_key in (
+ ("with_skill", "with_skill_summary"),
+ ("without_skill", "without_skill_summary"),
+ ):
+ if (
+ artifact_key in references
+ and isinstance(projected_job_failures.get(condition), str)
+ and projected_job_failures[condition]
+ ):
+ projected_job_failures[condition] = ""
+ omitted.append(f"job_failures.{condition}")
+ compact["job_failures"] = projected_job_failures
+
+ raw_agent_errors = compact.get("execution_errors")
+ if isinstance(raw_agent_errors, list) and raw_agent_errors and references:
+ agent_errors = [str(error) for error in raw_agent_errors if str(error)]
+ total = compact.get("execution_error_details_total")
+ exact_total = (
+ total
+ if isinstance(total, int) and not isinstance(total, bool) and total >= len(agent_errors)
+ else len(agent_errors)
+ )
+ compact["execution_errors"] = agent_errors[:1]
+ compact["execution_error_details_total"] = exact_total
+ compact["execution_error_details_shown"] = len(compact["execution_errors"])
+ compact["execution_error_details_truncated"] = len(compact["execution_errors"]) < exact_total
+ omitted.append("execution_errors")
+
+ if omitted:
+ compact["detail_projection"] = {
+ "artifacts": sorted(references),
+ "omitted_fields": sorted(set(omitted)),
+ }
+ return compact, references, sorted(set(omitted))
+
+
+def _persisted_result_projection(result: dict[str, Any], *, run_dir: Path) -> dict[str, Any]:
+ """Build a compact on-disk result while leaving the returned result complete."""
+ projected = dict(result)
+ projected_agents: dict[str, Any] = {}
+ projection_agents: dict[str, Any] = {}
+ omitted_fields: dict[str, list[str]] = {}
+ raw_agents = result.get("agents")
+ if isinstance(raw_agents, dict):
+ for agent, value in raw_agents.items():
+ compact, references, omitted = _compact_agent_result(str(agent), value, run_dir=run_dir)
+ projected_agents[str(agent)] = compact
+ if references:
+ projection_agents[str(agent)] = references
+ if omitted:
+ omitted_fields[str(agent)] = omitted
+ projected["agents"] = projected_agents
+
+ root_omitted: list[str] = []
+ has_summary_references = any(
+ "with_skill_summary" in references or "without_skill_summary" in references
+ for references in projection_agents.values()
+ )
+ raw_root_errors = projected.get("execution_errors")
+ if has_summary_references and isinstance(raw_root_errors, list):
+ root_errors, observed_total = _published_execution_errors(raw_root_errors)
+ declared_total = projected.get("execution_error_details_total")
+ exact_total = (
+ declared_total
+ if isinstance(declared_total, int)
+ and not isinstance(declared_total, bool)
+ and declared_total >= observed_total
+ else observed_total
+ )
+ compact_root_errors = root_errors[:1]
+ if compact_root_errors != raw_root_errors:
+ projected["execution_errors"] = compact_root_errors
+ root_omitted.append("execution_errors")
+ projected["execution_error_details_total"] = exact_total
+ projected["execution_error_details_shown"] = len(compact_root_errors)
+ projected["execution_error_details_truncated"] = len(compact_root_errors) < exact_total
+ if projected.get("error") != compact_root_errors:
+ projected["error"] = compact_root_errors
+ root_omitted.append("error")
+
+ projection = {
+ "schema_version": _FINAL_RESULT_PROJECTION_SCHEMA_VERSION,
+ "mode": "artifact_referenced" if omitted_fields or root_omitted else "inline",
+ "returned_result": "full",
+ "persisted_result": "compact" if omitted_fields or root_omitted else "inline",
+ "agents": projection_agents,
+ "omitted_detail_fields": omitted_fields,
+ "omitted_root_detail_fields": sorted(set(root_omitted)),
+ }
+ result["result_projection"] = projection
+ projected["result_projection"] = projection
+ return projected
+
+
+def _write_final_result(result_path: Path, result: dict[str, Any]) -> dict[str, Any]:
+ """Project, validate, and atomically publish the final result contract."""
+ inline_projection = {
+ "schema_version": _FINAL_RESULT_PROJECTION_SCHEMA_VERSION,
+ "mode": "inline",
+ "returned_result": "full",
+ "persisted_result": "inline",
+ "agents": {},
+ "omitted_detail_fields": {},
+ "omitted_root_detail_fields": [],
+ }
+ result["result_projection"] = inline_projection
+ full_encoded = _serialize_final_result(result)
+ try:
+ _validate_final_result_tree(result)
+ except _FinalResultContractError:
+ pass
+ else:
+ if len(full_encoded) <= FINAL_RESULT_MAX_BYTES:
+ write_output_file_atomically(result_path, full_encoded)
+ return result
+
+ projected = _persisted_result_projection(result, run_dir=result_path.parent)
+ encoded = _encode_final_result(projected)
+ write_output_file_atomically(result_path, encoded)
+ return projected
+
+
+def _serialized_json_bytes(value: object) -> int:
+ """Return bytes produced by the same JSON settings used for result files."""
+ return len(json.dumps(value, indent=2).encode("utf-8"))
+
+
+def _bounded_execution_error(value: object) -> str:
+ """Redact and bound one diagnostic by characters and serialized bytes."""
+ redacted = redact_sensitive_text(str(value))
+ safe = "".join(character if character.isprintable() else " " for character in redacted).strip()
+ if not safe:
+ return ""
+ if (
+ len(safe) <= PUBLISHED_EXECUTION_ERROR_MAX_CHARS
+ and _serialized_json_bytes(safe) <= PUBLISHED_EXECUTION_ERROR_MAX_SERIALIZED_BYTES
+ ):
+ return safe
+
+ maximum_prefix_chars = min(
+ len(safe),
+ PUBLISHED_EXECUTION_ERROR_MAX_CHARS - len(_EXECUTION_ERROR_TRUNCATION_MARKER),
+ )
+ lower = 0
+ upper = maximum_prefix_chars
+ while lower < upper:
+ middle = (lower + upper + 1) // 2
+ candidate = safe[:middle] + _EXECUTION_ERROR_TRUNCATION_MARKER
+ if _serialized_json_bytes(candidate) <= PUBLISHED_EXECUTION_ERROR_MAX_SERIALIZED_BYTES:
+ lower = middle
+ else:
+ upper = middle - 1
+ return safe[:lower] + _EXECUTION_ERROR_TRUNCATION_MARKER
+
+
+def _published_execution_errors(errors: list[object]) -> tuple[list[str], int]:
+ """Return a redacted, byte-bounded, de-duplicated diagnostic sample."""
+ published: list[str] = []
+ seen_details: set[str] = set()
+ seen_publications: set[str] = set()
+ for error in errors:
+ redacted = redact_sensitive_text(str(error))
+ detail = "".join(character if character.isprintable() else " " for character in redacted).strip()
+ if not detail or detail in seen_details:
+ continue
+ seen_details.add(detail)
+ safe = _bounded_execution_error(detail)
+ if safe in seen_publications:
+ continue
+ seen_publications.add(safe)
+ if (
+ len(published) < PUBLISHED_EXECUTION_ERRORS_MAX
+ and _serialized_json_bytes([*published, safe]) <= PUBLISHED_EXECUTION_ERRORS_MAX_SERIALIZED_BYTES
+ ):
+ published.append(safe)
+ return published, len(seen_details)
+
+
+def _merge_launch_execution_errors(result: dict[str, Any], launch_errors: list[object]) -> None:
+ """Overlay launch diagnostics without discarding hidden collector counts."""
+ raw_existing = result.get("execution_errors", [])
+ if isinstance(raw_existing, list):
+ existing: list[object] = raw_existing
+ elif raw_existing:
+ existing = [raw_existing]
+ else:
+ existing = []
+
+ _, observed_existing_total = _published_execution_errors(existing)
+ execution_errors, observed_combined_total = _published_execution_errors([*existing, *launch_errors])
+ new_launch_total = max(0, observed_combined_total - observed_existing_total)
+
+ raw_declared_total = result.get("execution_error_details_total")
+ if isinstance(raw_declared_total, int) and not isinstance(raw_declared_total, bool) and raw_declared_total >= 0:
+ declared_total = min(raw_declared_total, _MAX_JSON_SAFE_INTEGER)
+ else:
+ declared_total = observed_existing_total
+ declared_total = max(declared_total, observed_existing_total)
+
+ collector_truncated = result.get("execution_error_details_truncated") is True
+ error_total = min(_MAX_JSON_SAFE_INTEGER, declared_total + new_launch_total)
+
+ result["execution_status"] = "failed"
+ result["execution_errors"] = execution_errors
+ result["execution_error_details_total"] = error_total
+ result["execution_error_details_shown"] = len(execution_errors)
+ result["execution_error_details_truncated"] = collector_truncated or len(execution_errors) < error_total
+ # ``execution_errors`` is authoritative; ``error`` remains a compact
+ # list-shaped compatibility alias for older callers.
+ result["error"] = execution_errors[:1]
+
def _persist_dataset_truth(run_dir: Path, *, fallback_task_ids: list[str]) -> dict[str, Any]:
"""Persist immutable dataset and evaluator identity before staging cleanup."""
entries = load_staged_harbor_dataset(run_dir)
+ if getattr(entries, "_report_truncation", None):
+ raise DatasetSnapshotContractError
if not entries:
entries = [{"id": task_id} for task_id in fallback_task_ids]
- snapshot = build_dataset_snapshot(entries, evaluator_version=__version__)
+ try:
+ snapshot = build_dataset_snapshot(entries, evaluator_version=__version__)
+ encoded = encode_dataset_snapshot(snapshot)
+ except DatasetSnapshotContractError:
+ raise
+ except (MemoryError, RecursionError, TypeError, UnicodeError, ValueError):
+ raise DatasetSnapshotContractError from None
target = run_dir / "dataset_snapshot.json"
- with tempfile.NamedTemporaryFile(
- mode="w",
- encoding="utf-8",
- dir=run_dir,
- prefix=".dataset_snapshot.",
- suffix=".tmp",
- delete=False,
- ) as handle:
- temporary = Path(handle.name)
- json.dump(snapshot, handle, indent=2)
- handle.flush()
- os.fsync(handle.fileno())
- temporary.replace(target)
+ write_output_file_atomically(target, encoded)
return snapshot
_NVIDIA_BUILD_FILE_SENTINEL = "skillevaluator-file-backed-nvidia-key"
_NVIDIA_BUILD_KEY_FILE_ENV = "SKILLEVALUATOR_NVIDIA_API_KEY_FILE"
_NVIDIA_BUILD_BRIDGED_AGENT_DEFAULT_MODEL = "nvidia/nemotron-3-super-120b-a12b"
+_HARBOR_RUN_TIMEOUT_SECONDS = 7200.0
+_HARBOR_RUN_OUTPUT_MAX_BYTES = MAX_COMMAND_OUTPUT_BYTES
+_HARBOR_RUN_DIAGNOSTIC_TAIL_CHARS = 16 * 1024
+_HARBOR_RUN_OUTPUT_READ_BYTES = 64 * 1024
+_HARBOR_RUN_POLL_SECONDS = 0.01
+_HARBOR_RUN_TERMINATE_SECONDS = 0.1
+_HARBOR_RUN_REAP_SECONDS = 5.0
+
+
+@dataclass(frozen=True)
+class _BoundedHarborProcessResult:
+ returncode: int
+ output_tail: str
+ output_exceeded: bool
+
+
+class _HarborRunTimeoutError(RuntimeError):
+ """Raised after timed-out Harbor orchestration cleanup completes."""
+
+
+def _redact_harbor_diagnostic(detail: object, *, secret_values: set[str]) -> str:
+ """Normalize a complete diagnostic, then remove synthesized exact secrets."""
+ normalized = redact_progress_detail(detail, secret_values=secret_values)
+ exact_redactor = StreamingSecretRedactor(value for value in secret_values if len(value) >= 4)
+ return exact_redactor.feed(normalized) + exact_redactor.finish()
def _reserve_run_dir(results_root: Path, timestamp: str) -> Path:
@@ -138,160 +623,286 @@ def _reserve_run_dir(results_root: Path, timestamp: str) -> Path:
raise RuntimeError("Could not reserve a unique Tier 3 run directory")
-_HARBOR_BASE_ENV_VARS = frozenset(
+_TRUSTED_NETWORK_HOST_ENV_VARS = frozenset(
+ {
+ "ALL_PROXY",
+ "CURL_CA_BUNDLE",
+ "HTTPS_PROXY",
+ "HTTP_PROXY",
+ "NO_PROXY",
+ "REQUESTS_CA_BUNDLE",
+ "SSL_CERT_DIR",
+ "SSL_CERT_FILE",
+ "all_proxy",
+ "http_proxy",
+ "https_proxy",
+ "no_proxy",
+ }
+)
+_HARBOR_BASE_ENV_VARS = _TRUSTED_NETWORK_HOST_ENV_VARS | {
+ "COMSPEC",
+ "HOME",
+ "LANG",
+ "LC_ALL",
+ "LC_CTYPE",
+ "PATH",
+ "PATHEXT",
+ "SYSTEMROOT",
+ "TEMP",
+ "TMP",
+ "TMPDIR",
+ "USERPROFILE",
+ "WINDIR",
+ "XDG_RUNTIME_DIR",
+}
+_AWS_HOST_ENV_VARS = frozenset(
+ {
+ "AWS_ACCOUNT_ID",
+ "AWS_ACCOUNT_ID_ENDPOINT_MODE",
+ "AWS_ACCESS_KEY_ID",
+ "AWS_AUTH_SCHEME_PREFERENCE",
+ "AWS_CA_BUNDLE",
+ "AWS_CONFIG_FILE",
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN",
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI",
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
+ "AWS_CREDENTIAL_EXPIRATION",
+ "AWS_CREDENTIAL_FILE",
+ "AWS_CSM_CLIENT_ID",
+ "AWS_CSM_ENABLED",
+ "AWS_CSM_HOST",
+ "AWS_CSM_PORT",
+ "AWS_DATA_PATH",
+ "AWS_DEFAULT_PROFILE",
+ "AWS_DEFAULT_REGION",
+ "AWS_DEFAULTS_MODE",
+ "AWS_DISABLE_HOST_PREFIX_INJECTION",
+ "AWS_DISABLE_REQUEST_COMPRESSION",
+ "AWS_EC2_METADATA_DISABLED",
+ "AWS_EC2_METADATA_SERVICE_ENDPOINT",
+ "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
+ "AWS_EC2_METADATA_V1_DISABLED",
+ "AWS_ENDPOINT_DISCOVERY_ENABLED",
+ "AWS_ENDPOINT_URL",
+ "AWS_ENDPOINT_URL_SIGNIN",
+ "AWS_ENDPOINT_URL_SSO",
+ "AWS_ENDPOINT_URL_SSO_OIDC",
+ "AWS_ENDPOINT_URL_STS",
+ "AWS_EXECUTION_ENV",
+ "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS",
+ "AWS_IMDS_USE_IPV6",
+ "AWS_LOGIN_CACHE_DIRECTORY",
+ "AWS_MAX_ATTEMPTS",
+ "AWS_METADATA_SERVICE_NUM_ATTEMPTS",
+ "AWS_METADATA_SERVICE_TIMEOUT",
+ "AWS_NEW_RETRIES_2026",
+ "AWS_PROFILE",
+ "AWS_REGION",
+ "AWS_REQUEST_CHECKSUM_CALCULATION",
+ "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES",
+ "AWS_RESPONSE_CHECKSUM_VALIDATION",
+ "AWS_RETRY_MODE",
+ "AWS_ROLE_ARN",
+ "AWS_ROLE_SESSION_NAME",
+ "AWS_SDK_LOAD_CONFIG",
+ "AWS_SDK_UA_APP_ID",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_SECURITY_TOKEN",
+ "AWS_SESSION_TOKEN",
+ "AWS_SHARED_CREDENTIALS_FILE",
+ "AWS_SIGV4A_SIGNING_REGION_SET",
+ "AWS_STS_REGIONAL_ENDPOINTS",
+ "AWS_USE_DUALSTACK_ENDPOINT",
+ "AWS_USE_FIPS_ENDPOINT",
+ "AWS_WEB_IDENTITY_TOKEN_FILE",
+ "BOTOCORE_TCP_KEEPALIVE",
+ }
+)
+_EC2_HOST_ENV_VARS = _AWS_HOST_ENV_VARS | {"AWS_ENDPOINT_URL_EC2", "SSH_AUTH_SOCK"}
+_DOCKER_HOST_ENV_VARS = frozenset(
{
- "COMSPEC",
- "HOME",
- "LANG",
- "LC_ALL",
- "LC_CTYPE",
- "PATH",
- "PATHEXT",
- "SYSTEMROOT",
- "TEMP",
- "TMP",
- "TMPDIR",
- "WINDIR",
- "XDG_RUNTIME_DIR",
+ "COMPOSE_ANSI",
+ "COMPOSE_HTTP_TIMEOUT",
+ "COMPOSE_IGNORE_ORPHANS",
+ "COMPOSE_PARALLEL_LIMIT",
+ "COMPOSE_PROGRESS",
+ "COMPOSE_STATUS_STDOUT",
+ "DOCKER_API_VERSION",
+ "DOCKER_AUTH_CONFIG",
+ "DOCKER_CERT_PATH",
+ "DOCKER_CONFIG",
+ "DOCKER_CONTEXT",
+ "DOCKER_CUSTOM_HEADERS",
+ "DOCKER_DEFAULT_PLATFORM",
+ "DOCKER_HOST",
+ "DOCKER_TLS",
+ "DOCKER_TLS_VERIFY",
+ "SSH_AUTH_SOCK",
}
)
_HARBOR_ENV_MODE_VARS = {
- "docker": frozenset(
- {
- "DOCKER_API_VERSION",
- "DOCKER_CERT_PATH",
- "DOCKER_CONFIG",
- "DOCKER_CONTEXT",
- "DOCKER_HOST",
- "DOCKER_TLS_VERIFY",
- }
- ),
+ "docker": _DOCKER_HOST_ENV_VARS,
"daytona": frozenset(
{
"DAYTONA_API_KEY",
"DAYTONA_API_URL",
+ "DAYTONA_HAPPY_EYEBALLS_DELAY",
"DAYTONA_JWT_TOKEN",
"DAYTONA_ORGANIZATION_ID",
+ "DAYTONA_SERVER_URL",
"DAYTONA_TARGET",
}
),
- "e2b": frozenset({"E2B_API_KEY"}),
- "modal": frozenset({"MODAL_ENVIRONMENT", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"}),
- "runloop": frozenset({"RUNLOOP_API_KEY"}),
+ "e2b": frozenset({"E2B_API_KEY", "E2B_API_URL", "E2B_DOMAIN", "E2B_SANDBOX_URL"}),
+ "modal": frozenset(
+ {
+ "MODAL_CONFIG_PATH",
+ "MODAL_ENVIRONMENT",
+ "MODAL_OVERRIDE_HEADERS",
+ "MODAL_PROFILE",
+ "MODAL_SERVER_URL",
+ "MODAL_TOKEN_ID",
+ "MODAL_TOKEN_SECRET",
+ }
+ ),
+ "runloop": frozenset({"RUNLOOP_API_KEY", "RUNLOOP_BASE_URL", "RUNLOOP_CUSTOM_HEADERS"}),
"langsmith": frozenset(
{
"LANGCHAIN_API_KEY",
+ "LANGCHAIN_ENDPOINT",
"LANGSMITH_API_KEY",
+ "LANGSMITH_CONFIG_FILE",
"LANGSMITH_ENDPOINT",
"LANGSMITH_PROFILE",
"LANGSMITH_SANDBOX_API_URL",
+ "LANGSMITH_WORKSPACE_ID",
}
),
+ "ec2": _EC2_HOST_ENV_VARS,
"gke": frozenset(
- {"CLOUDSDK_CONFIG", "GCP_PROJECT", "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT", "KUBECONFIG"}
+ {
+ "CLOUDSDK_ACTIVE_CONFIG_NAME",
+ "CLOUDSDK_AUTH_ACCESS_TOKEN",
+ "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE",
+ "CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT",
+ "CLOUDSDK_CONFIG",
+ "CLOUDSDK_CORE_ACCOUNT",
+ "CLOUDSDK_CORE_PROJECT",
+ "GCP_PROJECT",
+ "GOOGLE_APPLICATION_CREDENTIALS",
+ "GOOGLE_CLOUD_PROJECT",
+ "GOOGLE_CLOUD_QUOTA_PROJECT",
+ "KUBECONFIG",
+ }
+ ),
+ "ack": _DOCKER_HOST_ENV_VARS
+ | {
+ "KUBECONFIG",
+ "KUBERNETES_SERVICE_HOST",
+ "KUBERNETES_SERVICE_PORT",
+ },
+ "openshift": frozenset({"KUBECONFIG"}),
+ "novita": frozenset(
+ {
+ "NOVITA_ACCESS_TOKEN",
+ "NOVITA_API_KEY",
+ "NOVITA_API_URL",
+ "NOVITA_BASE_URL",
+ "NOVITA_DOMAIN",
+ "NOVITA_SANDBOX_URL",
+ }
+ ),
+ "apple-container": frozenset(),
+ "singularity": frozenset(
+ {
+ "APPTAINER_AUTHFILE",
+ "APPTAINER_CONFIGDIR",
+ "APPTAINER_DOCKER_PASSWORD",
+ "APPTAINER_DOCKER_USERNAME",
+ "SINGULARITY_AUTHFILE",
+ "SINGULARITY_CONFIGDIR",
+ "SINGULARITY_DOCKER_PASSWORD",
+ "SINGULARITY_DOCKER_USERNAME",
+ }
),
- "novita": frozenset({"NOVITA_API_KEY", "NOVITA_API_URL", "NOVITA_BASE_URL", "NOVITA_DOMAIN"}),
"islo": frozenset({"ISLO_API_KEY", "ISLO_API_URL", "ISLO_COMPUTE_URL"}),
- "tensorlake": frozenset({"TENSORLAKE_API_KEY"}),
- "cwsandbox": frozenset({"CWSANDBOX_API_KEY"}),
- "wandb": frozenset({"WANDB_API_KEY", "WANDB_BASE_URL"}),
+ "tensorlake": frozenset(
+ {
+ "TENSORLAKE_API_KEY",
+ "TENSORLAKE_API_URL",
+ "TENSORLAKE_ORGANIZATION_ID",
+ "TENSORLAKE_PAT",
+ "TENSORLAKE_PROJECT_ID",
+ "TENSORLAKE_SANDBOX_PROXY_URL",
+ }
+ ),
+ "cwsandbox": frozenset({"CWSANDBOX_API_KEY", "CWSANDBOX_BASE_URL"}),
+ "wandb": frozenset({"NETRC", "WANDB_API_KEY", "WANDB_BASE_URL", "WANDB_ENTITY", "WANDB_PROJECT"}),
"use-computer": frozenset(
{"USE_COMPUTER_API_KEY", "USE_COMPUTER_HOST", "USE_COMPUTER_SNAPSHOT", "USE_COMPUTER_VERSION"}
),
-}
-_BEDROCK_HOST_ENV_VARS = frozenset(
- {
- "AWS_ACCESS_KEY_ID",
- "AWS_BEARER_TOKEN_BEDROCK",
- "AWS_CONFIG_FILE",
- "AWS_CONTAINER_AUTHORIZATION_TOKEN",
- "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
- "AWS_CONTAINER_CREDENTIALS_FULL_URI",
- "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
- "AWS_DEFAULT_REGION",
- "AWS_PROFILE",
- "AWS_ROLE_ARN",
- "AWS_ROLE_SESSION_NAME",
- "AWS_SDK_LOAD_CONFIG",
- "AWS_SECRET_ACCESS_KEY",
- "AWS_SESSION_TOKEN",
- "AWS_SHARED_CREDENTIALS_FILE",
- "AWS_WEB_IDENTITY_TOKEN_FILE",
- }
-)
-_RUNTIME_ENV_HOST_CONTROL_NAMES = (
- frozenset(
+ "cua-cloud": frozenset(
{
- "ALL_PROXY",
- "BASHOPTS",
- "BASH_ENV",
- "CDPATH",
- "CLAUDE_CODE_DISABLE_POLICY_SKILLS",
- "CLAUDE_CONFIG_DIR",
- "CLASSPATH",
- "COMSPEC",
- "CODEX_HOME",
- "ENV",
- "GCONV_PATH",
- "GEMINI_CLI_HOME",
- "HOME",
- "HOSTALIASES",
- "HTTPS_PROXY",
- "HTTP_PROXY",
- "IFS",
- "JAVA_TOOL_OPTIONS",
- "LOCPATH",
- "LUA_CPATH",
- "LUA_INIT",
- "LUA_PATH",
- "NLSPATH",
- "NO_PROXY",
- "OPENCODE_CONFIG_DIR",
- "PATHEXT",
- "PATH",
- "PERL5LIB",
- "PERL5OPT",
- "REQUESTS_CA_BUNDLE",
- "RES_OPTIONS",
- "RUBYOPT",
- "RUBYLIB",
- "SHELLOPTS",
- "SSLKEYLOGFILE",
- "SSL_CERT_DIR",
- "SSL_CERT_FILE",
- "SSH_AUTH_SOCK",
- "SYSTEMROOT",
- "TEMP",
- "TMP",
- "TMPDIR",
- "USERPROFILE",
- "WINDIR",
- "XDG_CONFIG_HOME",
- "XDG_RUNTIME_DIR",
- "ZDOTDIR",
- "_JAVA_OPTIONS",
+ "CUA_BASE_URL",
+ "CUA_CLIENT_ID",
+ "CUA_CLIENT_SECRET",
+ "CUA_CLOUD_NAMESPACE",
+ "CUA_CLOUD_STARTUP_COMMAND",
+ "CUA_TOKEN_URL",
}
- )
+ ),
+ "blaxel": frozenset(
+ {
+ "BL_API_KEY",
+ "BL_API_VERSION",
+ "BL_CLIENT_CREDENTIALS",
+ "BL_ENV",
+ "BL_REGION",
+ "BL_WORKSPACE",
+ }
+ ),
+ "opensandbox": frozenset({"OPENSANDBOX_API_KEY", "OPENSANDBOX_DOMAIN"}),
+ "beam": frozenset(
+ {
+ "API_HOST",
+ "API_PORT",
+ "BEAM_TOKEN",
+ "GATEWAY_HOST",
+ "GATEWAY_PORT",
+ "INTERNAL_API_HOST",
+ "INTERNAL_API_PORT",
+ "REALTIME_HOST",
+ }
+ ),
+ "skypilot": _DOCKER_HOST_ENV_VARS
+ | frozenset(
+ {
+ "HARBOR_SKYPILOT_REGISTRY",
+ "SKYPILOT_API_SERVER_ENDPOINT",
+ "SKYPILOT_GLOBAL_CONFIG",
+ "SKYPILOT_PROJECT_CONFIG",
+ "SKYPILOT_SERVICE_ACCOUNT_TOKEN",
+ }
+ ),
+ "hf-sandbox": frozenset({"HF_ENDPOINT", "HF_HOME", "HF_TOKEN", "HF_TOKEN_PATH", "HUGGING_FACE_HUB_TOKEN"}),
+ "hyperbrowser": _DOCKER_HOST_ENV_VARS | frozenset({"HYPERBROWSER_API_KEY", "HYPERBROWSER_BASE_URL"}),
+ "vercel": frozenset({"VERCEL_OIDC_TOKEN", "VERCEL_PROJECT_ID", "VERCEL_TEAM_ID", "VERCEL_TOKEN"}),
+}
+_BEDROCK_HOST_ENV_VARS = _AWS_HOST_ENV_VARS | {
+ "AWS_BEARER_TOKEN_BEDROCK",
+ "AWS_CSM_ENABLED",
+ "AWS_CSM_PORT",
+ "AWS_ENDPOINT_URL_BEDROCK",
+ "AWS_ENDPOINT_URL_BEDROCK_RUNTIME",
+}
+_RUNTIME_ENV_HOST_CONTROL_NAMES = (
+ _RUNTIME_PROCESS_CONTROL_ENV_NAMES
| _BEDROCK_HOST_ENV_VARS
- | _VERIFIER_JUDGE_MODEL_ENV_VARS
+ | _VERIFIER_JUDGE_CONTROL_ENV_VARS
| frozenset().union(*_HARBOR_ENV_MODE_VARS.values())
)
-_RUNTIME_ENV_HOST_CONTROL_PREFIXES = (
- "BASH_FUNC_",
- "COMPOSE_",
- "DOCKER_",
- "DYLD_",
- "GIT_",
- "HARBOR_",
- "LD_",
- "NODE_",
- "OTEL_",
- "PIP_",
- "PYTHON",
- "SKILL_EVAL_",
- "SKILLEVALUATOR_",
- "UV_",
-)
+_RUNTIME_ENV_HOST_CONTROL_PREFIXES = _RUNTIME_PROCESS_CONTROL_ENV_PREFIXES
_OPERATOR_OWNED_AGENT_ENV = frozenset(
{
"ANTHROPIC_API_KEY",
@@ -322,7 +933,7 @@ def _harbor_bin() -> str:
def _harbor_supports_yes() -> bool:
- """Harbor 0.13.2, the supported Tier 3 dependency, accepts ``--yes``."""
+ """The supported Harbor CLI accepts ``--yes`` for non-interactive runs."""
return True
@@ -363,6 +974,117 @@ def _nvidia_build_key_handoff(
return _NvidiaBuildKeyHandoff(subprocess_env)
+_HARBOR_RUNTIME_POLICY_KWARGS = frozenset(
+ {
+ "context_id",
+ "cpu_enforcement_policy",
+ "delete",
+ "environment_dir",
+ "environment_name",
+ "extra_allowed_hosts",
+ "extra_docker_compose",
+ "force_build",
+ "keep_containers",
+ "logger",
+ "memory_enforcement_policy",
+ "mounts",
+ "mounts_json",
+ "network_policy",
+ "override_cpus",
+ "override_gpus",
+ "override_memory_mb",
+ "override_storage_mb",
+ "override_tpu",
+ "persistent_env",
+ "pod_capabilities_add",
+ "pod_capabilities_drop",
+ "pod_overrides",
+ "pod_privileged",
+ "pod_run_as_group",
+ "pod_run_as_user",
+ "phase_network_policies",
+ "session_id",
+ "suppress_override_warnings",
+ "extra_env",
+ "extra_volume_mounts",
+ "extra_volumes",
+ "init_containers",
+ "task_env_config",
+ "trial_paths",
+ }
+)
+
+_HARBOR_ENVIRONMENT_RUNTIME_POLICY_KWARGS: dict[str, frozenset[str]] = {
+ "ack": frozenset(
+ {
+ "build_job_namespace",
+ "buildkit_address",
+ "dind_image",
+ "memory_limit_multiplier",
+ "pod_annotations",
+ "pod_labels",
+ "sandbox_env_vars",
+ "service_account",
+ "use_buildkit",
+ }
+ ),
+ "blaxel": frozenset({"dind_extra_args"}),
+ "cua-cloud": frozenset({"claim_spec"}),
+ "daytona": frozenset({"network_block_all"}),
+ "ec2": frozenset({"iam_instance_profile", "strict_host_key_checking"}),
+ "gke": frozenset({"memory_limit_multiplier"}),
+ "modal": frozenset({"volumes"}),
+ "opensandbox": frozenset({"volumes"}),
+ "openshift": frozenset({"service_account_name"}),
+ "singularity": frozenset({"singularity_no_mount"}),
+ "use-computer": frozenset({"resources"}),
+ "vercel": frozenset({"ports"}),
+}
+
+
+def _environment_kwarg_policy_error(env_mode: str, environment_kwargs: Mapping[str, Any]) -> str | None:
+ if not environment_kwargs:
+ return None
+ if env_mode == "docker":
+ return "Environment kwargs are not supported for SkillEvaluator Docker mode"
+ if env_mode == ENV_MODE_LOCAL:
+ return "Environment kwargs are not supported for SkillEvaluator local mode"
+ reserved = _HARBOR_RUNTIME_POLICY_KWARGS | _HARBOR_ENVIRONMENT_RUNTIME_POLICY_KWARGS.get(env_mode, frozenset())
+ if collisions := sorted(reserved & environment_kwargs.keys()):
+ return "Environment kwarg(s) reserved for Harbor runtime policy: " + ", ".join(collisions)
+ if unknown := sorted(environment_kwargs.keys() - HARBOR_V022_ENVIRONMENT_KWARGS[env_mode]):
+ return f"Harbor 0.22.0 environment '{env_mode}' does not accept environment kwarg(s): " + ", ".join(unknown)
+ return None
+
+
+def _validated_timeout_multiplier(value: object) -> float:
+ """Return one finite positive timeout scale accepted by every entry point."""
+ if isinstance(value, bool) or not isinstance(value, int | float):
+ raise ValueError("timeout_multiplier must be a finite number greater than 0")
+ try:
+ normalized = float(value)
+ except OverflowError:
+ raise ValueError("timeout_multiplier must be a finite number greater than 0") from None
+ if not math.isfinite(normalized) or normalized <= 0:
+ raise ValueError("timeout_multiplier must be a finite number greater than 0")
+ if normalized > MAX_HARBOR_TIMEOUT_MULTIPLIER:
+ raise ValueError("timeout_multiplier must yield finite Harbor timeouts")
+ return normalized
+
+
+def _validated_pass_threshold(value: object) -> float:
+ """Return one finite unit-interval attempt threshold."""
+ if isinstance(value, bool) or not isinstance(value, int | float):
+ raise ValueError("pass_threshold must be a finite number between 0.0 and 1.0")
+ try:
+ normalized = float(value)
+ except OverflowError:
+ raise ValueError("pass_threshold must be a finite number between 0.0 and 1.0") from None
+ if not math.isfinite(normalized) or not 0.0 <= normalized <= 1.0:
+ raise ValueError("pass_threshold must be a finite number between 0.0 and 1.0")
+ return normalized
+
+
def build_harbor_run_command(
*,
dataset_path: str | Path,
@@ -381,12 +1103,20 @@ def build_harbor_run_command(
override_storage_mb: int | None = None,
agent_import_path: str | None = None,
verifier_env: Mapping[str, str] | None = None,
+ environment_kwargs: Mapping[str, Any] | None = None,
) -> list[str]:
"""Build a Harbor invocation for a built-in environment type or local mode."""
if env_mode not in HARBOR_ENV_MODES:
raise ValueError(f"env_mode must be one of: {', '.join(sorted(HARBOR_ENV_MODES))}")
+ timeout_multiplier = _validated_timeout_multiplier(timeout_multiplier)
if agent_import_path and env_mode not in {"docker", ENV_MODE_LOCAL}:
raise ValueError("agent_import_path is supported only with --env docker or local")
+ validated_environment_kwargs = validate_environment_kwargs(
+ dict(environment_kwargs or {}),
+ env_mode=env_mode,
+ )
+ if policy_error := _environment_kwarg_policy_error(env_mode, validated_environment_kwargs):
+ raise ValueError(policy_error)
command = [
_harbor_bin(),
@@ -402,19 +1132,18 @@ def build_harbor_run_command(
]
if env_mode == ENV_MODE_LOCAL:
# Local mode is a custom SkillEvaluator environment + agent wrappers,
- # dispatched via import paths (not Harbor's --env), with sandbox knobs
- # passed as environment-kwargs (--ek). Harbor's create_agent_from_config
- # prefers the agent NAME when both -a and --agent-import-path are set, so
- # local mode passes ONLY --agent-import-path (its wrapper skips the
- # Debian apt-get bootstrap the stock agent runs) and never -a.
+ # dispatched as import paths through Harbor's unified --agent/--env
+ # flags, with sandbox knobs passed as environment-kwargs (--ek). The
+ # custom agent wrapper skips the Debian apt-get bootstrap used by the
+ # stock agent.
from skillevaluator.tier3.harbor import LOCAL_AGENT_IMPORT_PATHS, LOCAL_ENV_IMPORT_PATH, local_sandbox
from skillevaluator.tier3.harbor.local_runtime import default_runtime_root
agent_import_path = agent_import_path or LOCAL_AGENT_IMPORT_PATHS.get(agent)
if not agent_import_path:
raise ValueError(f"--env-mode local does not support agent: {agent}")
- command.extend(["--agent-import-path", agent_import_path])
- command.extend(["--environment-import-path", LOCAL_ENV_IMPORT_PATH])
+ command.extend(["--agent", agent_import_path])
+ command.extend(["--env", LOCAL_ENV_IMPORT_PATH])
command.extend(["--ek", f"runtime_root={default_runtime_root()}"])
command.extend(["--ek", f"runtime_agent={agent}"])
command.extend(["--ek", f"sandbox_mode={local_sandbox.resolve_mode(None)}"])
@@ -438,12 +1167,14 @@ def build_harbor_run_command(
)
elif env_mode == "docker":
if agent_import_path:
- command.extend(["--agent-import-path", agent_import_path])
+ command.extend(["--agent", agent_import_path])
else:
- command.extend(["-a", agent])
- command.extend(["--environment-import-path", SECURE_DOCKER_ENV_IMPORT_PATH])
+ command.extend(["--agent", agent])
+ command.extend(["--env", SECURE_DOCKER_ENV_IMPORT_PATH])
else:
- command.extend(["-a", agent, "--env", env_mode])
+ command.extend(["--agent", agent, "--env", env_mode])
+ for name, value in sorted(validated_environment_kwargs.items()):
+ command.extend(["--ek", encode_environment_kwarg(name, value)])
if jobs_dir is not None:
command.extend(["--jobs-dir", str(jobs_dir)])
if disable_verification:
@@ -474,7 +1205,7 @@ def _provider_environment(config: ProviderConfig) -> dict[str, str]:
"SKILL_EVAL_LLM_MODEL": config.model,
}
environment.update(
- {name: value for name in _VERIFIER_JUDGE_MODEL_ENV_VARS if (value := os.environ.get(name, "").strip())}
+ {name: value for name in _VERIFIER_JUDGE_CONTROL_ENV_VARS if (value := os.environ.get(name, "").strip())}
)
if config.provider == "anthropic":
environment["ANTHROPIC_API_KEY"] = config.api_key or ""
@@ -672,13 +1403,208 @@ def _validate_agent_provider_credentials(
return []
+def _environment_kwarg_prerequisite_errors(
+ env_mode: str,
+ environment_kwargs: Mapping[str, Any] | None,
+) -> list[str]:
+ """Validate constructor requirements Harbor cannot check in ``preflight``."""
+ try:
+ kwargs = validate_environment_kwargs(
+ dict(environment_kwargs or {}),
+ env_mode=env_mode,
+ )
+ except (RecursionError, TypeError, ValueError) as exc:
+ return [f"Invalid --environment-kwarg: {exc}"]
+ if policy_error := _environment_kwarg_policy_error(env_mode, kwargs):
+ return [policy_error]
+
+ def invalid_strings(*names: str) -> list[str]:
+ return [name for name in names if not isinstance(kwargs.get(name), str) or not kwargs[name].strip()]
+
+ required: tuple[str, ...] = ()
+ if env_mode == "gke":
+ required = ("cluster_name", "region", "namespace", "registry_location", "registry_name")
+ elif env_mode == "ack":
+ required = ("namespace",)
+ elif env_mode == "ec2":
+ required = ("region",)
+ if invalid := invalid_strings(*required):
+ return [
+ f"Harbor environment '{env_mode}' requires non-empty string --environment-kwarg for: " + ", ".join(invalid)
+ ]
+ if env_mode == "ack" and (
+ invalid := invalid_strings(*(name for name in ("context", "kubeconfig") if name in kwargs))
+ ):
+ return ["Harbor environment 'ack' requires non-empty string --environment-kwarg for: " + ", ".join(invalid)]
+ if env_mode == "opensandbox" and kwargs.get("domain") is not None and invalid_strings("domain"):
+ return ["Harbor environment 'opensandbox' requires domain to be a non-empty string when provided"]
+ if env_mode == "ec2":
+ launch_mode_value = kwargs.get("launch_mode", "ephemeral")
+ if not isinstance(launch_mode_value, str):
+ return ["Harbor environment 'ec2' requires launch_mode to be 'ephemeral' or 'attach'"]
+ launch_mode = launch_mode_value
+ if launch_mode not in {"ephemeral", "attach"}:
+ return ["Harbor environment 'ec2' requires launch_mode to be 'ephemeral' or 'attach'"]
+ conditional = "ami_id" if launch_mode == "ephemeral" else "instance_id"
+ if invalid_strings(conditional):
+ return [
+ f"Harbor environment 'ec2' launch_mode={launch_mode!r} requires --environment-kwarg {conditional}=VALUE"
+ ]
+ if (ssh_key_path := kwargs.get("ssh_key_path")) is not None:
+ try:
+ ssh_key_exists = isinstance(ssh_key_path, str) and Path(ssh_key_path).expanduser().is_file()
+ except (OSError, RuntimeError):
+ ssh_key_exists = False
+ if not ssh_key_exists:
+ return ["Harbor environment 'ec2' requires ssh_key_path to name an existing regular file"]
+ if launch_mode == "ephemeral" and kwargs.get("use_public_ip") is False and invalid_strings("subnet_id"):
+ return ["Harbor environment 'ec2' use_public_ip=False requires a non-empty subnet_id"]
+ return []
+
+
+def _environment_extra_install_hint(env_mode: str) -> str:
+ extra = HARBOR_ENVIRONMENT_EXTRAS.get(env_mode)
+ if extra is not None:
+ return f"Install 'harbor[{extra}]==0.22.0'."
+ system_hints = {
+ "apple-container": "Install the Apple container CLI; Harbor has no Python extra for this backend.",
+ "openshift": "Install the OpenShift oc CLI; Harbor has no Python extra for this backend.",
+ "singularity": "Install the singularity CLI; Harbor has no Python extra for this backend.",
+ }
+ return system_hints.get(env_mode, "Reinstall SkillEvaluator with its Tier 3 extra.")
+
+
+def _check_ack_cluster_readiness(environment_kwargs: Mapping[str, Any]) -> None:
+ """Load ACK credentials and run a bounded namespaced pod-list probe."""
+ from kubernetes import client as k8s_client
+ from kubernetes import config as k8s_config
+
+ load_kwargs: dict[str, str] = {}
+ if context := environment_kwargs.get("context"):
+ load_kwargs["context"] = str(context)
+ if kubeconfig := environment_kwargs.get("kubeconfig"):
+ load_kwargs["config_file"] = str(kubeconfig)
+ try:
+ k8s_config.load_kube_config(**load_kwargs)
+ except k8s_config.ConfigException:
+ k8s_config.load_incluster_config()
+
+ api_client = k8s_client.ApiClient()
+ try:
+ core_api = k8s_client.CoreV1Api(api_client)
+ core_api.list_namespaced_pod(
+ namespace=str(environment_kwargs["namespace"]),
+ limit=1,
+ _request_timeout=(5, 10),
+ )
+ finally:
+ api_client.close()
+
+
+_ACK_CLUSTER_READINESS_SUBPROCESS_TIMEOUT_SECONDS = 20
+_ACK_CLUSTER_READINESS_REAP_TIMEOUT_SECONDS = 5
+_ACK_CLUSTER_READINESS_PROBE_CODE = """\
+import json
+import sys
+
+from skillevaluator.tier3.harbor.runner import _check_ack_cluster_readiness
+
+try:
+ _check_ack_cluster_readiness(json.loads(sys.stdin.read()))
+except Exception as exc:
+ sys.stderr.write((str(exc) or type(exc).__name__)[:4096])
+ raise SystemExit(1) from None
+"""
+
+
+def _terminate_ack_readiness_process(process: subprocess.Popen[str]) -> None:
+ """Kill the ACK probe and its exec-auth descendants, then reap it."""
+ if os.name == "posix":
+ with suppress(ProcessLookupError):
+ os.killpg(process.pid, signal.SIGKILL)
+ else:
+ process.kill()
+ try:
+ process.wait(timeout=_ACK_CLUSTER_READINESS_REAP_TIMEOUT_SECONDS)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ process.wait(timeout=_ACK_CLUSTER_READINESS_REAP_TIMEOUT_SECONDS)
+
+
+def _check_ack_cluster_readiness_subprocess(
+ environment_kwargs: Mapping[str, Any],
+ *,
+ subprocess_env: Mapping[str, str],
+) -> None:
+ """Run ACK's bounded pod-list probe under Harbor's exact child environment."""
+ validated = validate_environment_kwargs(dict(environment_kwargs), env_mode="ack")
+ if policy_error := _environment_kwarg_policy_error("ack", validated):
+ raise ValueError(policy_error)
+ payload = {name: validated[name] for name in ("namespace", "context", "kubeconfig") if name in validated}
+ encoded = json.dumps(payload, ensure_ascii=True, separators=(",", ":"), allow_nan=False)
+ child_env = dict(subprocess_env)
+ redacted_values = secret_values_from_environment(child_env)
+ redacted_values.update(str(value) for value in payload.values() if isinstance(value, str) and value)
+ process: subprocess.Popen[str] | None = None
+ try:
+ process = subprocess.Popen(
+ [sys.executable, "-c", _ACK_CLUSTER_READINESS_PROBE_CODE],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ env=child_env,
+ start_new_session=os.name == "posix",
+ )
+ stdout, stderr = process.communicate(
+ encoded,
+ timeout=_ACK_CLUSTER_READINESS_SUBPROCESS_TIMEOUT_SECONDS,
+ )
+ except subprocess.TimeoutExpired:
+ assert process is not None
+ _terminate_ack_readiness_process(process)
+ raise RuntimeError(
+ "ACK namespaced pod-list readiness probe timed out after "
+ f"{_ACK_CLUSTER_READINESS_SUBPROCESS_TIMEOUT_SECONDS} seconds"
+ ) from None
+ except OSError as exc:
+ if process is not None and process.returncode is None:
+ _terminate_ack_readiness_process(process)
+ detail = redact_progress_detail(exc, secret_values=redacted_values) or type(exc).__name__
+ raise RuntimeError(f"ACK namespaced pod-list readiness probe could not start: {detail}") from None
+ if process.returncode == 0:
+ return
+ output = "\n".join(part for part in (stderr, stdout) if part).strip()
+ detail = redact_progress_detail(output, secret_values=redacted_values) or f"probe exited {process.returncode}"
+ raise RuntimeError(f"ACK namespaced pod-list readiness probe failed: {detail[-2000:]}")
+
+
+def _modal_custom_config_status() -> tuple[bool, str | None]:
+ raw = os.environ.get("MODAL_CONFIG_PATH")
+ if raw is None:
+ return False, None
+ if not raw.strip():
+ return False, "MODAL_CONFIG_PATH must name an existing regular file."
+ try:
+ is_file = Path(raw).expanduser().is_file()
+ except (OSError, RuntimeError):
+ is_file = False
+ if not is_file:
+ return False, "MODAL_CONFIG_PATH must name an existing regular file."
+ return True, None
+
+
def _check_prerequisites(
env_mode: str = DEFAULT_ENV_MODE,
agents: list[str] | None = None,
+ environment_kwargs: Mapping[str, Any] | None = None,
+ subprocess_env: Mapping[str, str] | None = None,
) -> list[str]:
"""Check Harbor and the selected environment (built-in or local mode)."""
if env_mode not in HARBOR_ENV_MODES:
return [f"Unsupported Harbor environment '{env_mode}'. Choose one of: {', '.join(sorted(HARBOR_ENV_MODES))}"]
+ if kwarg_errors := _environment_kwarg_prerequisite_errors(env_mode, environment_kwargs):
+ return kwarg_errors
if env_mode == ENV_MODE_LOCAL:
from skillevaluator.tier3.harbor import local_sandbox
@@ -693,6 +1619,38 @@ def _check_prerequisites(
'uv tool install "skillevaluator[all] @ git+https://github.com/NVIDIA/SkillEvaluator.git"'
]
+ if env_mode == "singularity" and shutil.which("singularity") is None:
+ return ["Harbor environment 'singularity' requires the singularity CLI on PATH."]
+ if env_mode == "islo" and not os.environ.get("ISLO_API_KEY", "").strip():
+ return ["Harbor environment 'islo' requires a non-empty ISLO_API_KEY in the host environment."]
+ if env_mode == "opensandbox" and (environment_kwargs or {}).get("domain") is None:
+ opensandbox_env = (
+ subprocess_env
+ if subprocess_env is not None
+ else _selected_host_environment(
+ _HARBOR_BASE_ENV_VARS | _HARBOR_ENV_MODE_VARS["opensandbox"],
+ os.environ,
+ )
+ )
+ if not opensandbox_env.get("OPENSANDBOX_DOMAIN", "").strip():
+ return [
+ "Harbor environment 'opensandbox' requires a non-empty domain --environment-kwarg "
+ "or child-visible OPENSANDBOX_DOMAIN."
+ ]
+
+ if env_mode == "modal":
+ _, modal_config_error = _modal_custom_config_status()
+ if modal_config_error:
+ return [modal_config_error]
+ try:
+ modal_spec = importlib.util.find_spec("modal")
+ except (ImportError, ValueError):
+ modal_spec = None
+ if modal_spec is None:
+ return [
+ f"Harbor environment 'modal' needs optional dependencies. {_environment_extra_install_hint(env_mode)}"
+ ]
+
if env_mode == ENV_MODE_LOCAL:
# Local mode is a host sandbox, not a Harbor-native backend: verify the
# OS sandbox is usable and the requested agent CLIs are installed.
@@ -727,10 +1685,18 @@ def _check_prerequisites(
check=False,
)
except (OSError, subprocess.SubprocessError) as exc:
- return [f"Docker Compose v2 is required for Tier 3 Docker mode: {exc}"]
+ detail = redact_progress_detail(
+ exc,
+ secret_values=secret_values_from_environment(os.environ),
+ )
+ return [f"Docker Compose v2 is required for Tier 3 Docker mode: {detail}"]
if compose.returncode != 0:
detail = (compose.stderr or compose.stdout).strip()
- suffix = f": {detail}" if detail else ""
+ safe_detail = redact_progress_detail(
+ detail,
+ secret_values=secret_values_from_environment(os.environ),
+ )
+ suffix = f": {safe_detail}" if safe_detail else ""
return [f"Docker Compose v2 is required for Tier 3 Docker mode{suffix}"]
try:
@@ -738,23 +1704,51 @@ def _check_prerequisites(
from harbor.models.environment_type import EnvironmentType
EnvironmentFactory.run_preflight(EnvironmentType(env_mode))
+ if env_mode == "ack":
+ ack_subprocess_env = (
+ dict(subprocess_env)
+ if subprocess_env is not None
+ else _selected_host_environment(
+ _HARBOR_BASE_ENV_VARS | _HARBOR_ENV_MODE_VARS["ack"],
+ os.environ,
+ )
+ )
+ _check_ack_cluster_readiness_subprocess(
+ dict(environment_kwargs or {}),
+ subprocess_env=ack_subprocess_env,
+ )
except ImportError as exc:
+ detail = redact_progress_detail(
+ exc,
+ secret_values=secret_values_from_environment(os.environ),
+ )
return [
- f"Harbor environment '{env_mode}' needs optional dependencies: {exc}. "
- "Install the matching Harbor environment extra."
+ f"Harbor environment '{env_mode}' needs optional dependencies: {detail}. "
+ f"{_environment_extra_install_hint(env_mode)}"
]
except SystemExit as exc:
- detail = " ".join(str(exc).split()) or "preflight exited without a diagnostic"
+ detail = (
+ redact_progress_detail(
+ exc,
+ secret_values=secret_values_from_environment(os.environ),
+ )
+ or "preflight exited without a diagnostic"
+ )
return [f"Harbor environment '{env_mode}' is not ready: {detail}"]
except Exception as exc:
- return [f"Harbor environment '{env_mode}' is not ready: {exc}"]
+ detail = redact_progress_detail(
+ exc,
+ secret_values=secret_values_from_environment(os.environ),
+ )
+ return [f"Harbor environment '{env_mode}' is not ready: {detail}"]
return []
def _is_operator_owned_runtime_name(name: str) -> bool:
normalized = name.upper()
return (
- normalized in _RUNTIME_ENV_HOST_CONTROL_NAMES
+ is_sensitive_key(name)
+ or normalized in _RUNTIME_ENV_HOST_CONTROL_NAMES
or normalized in _OPERATOR_OWNED_AGENT_ENV
or normalized.startswith(_RUNTIME_ENV_HOST_CONTROL_PREFIXES)
)
@@ -901,23 +1895,37 @@ def _job_judge_override(provider_env: Mapping[str, str], grading_mode: str) -> t
def _job_judge_verifier_env(provider_env: Mapping[str, str], grading_mode: str) -> dict[str, str]:
"""Return placeholder-based judge overrides for Harbor's verifier job layer."""
- selected = _job_judge_override(provider_env, grading_mode)
- if selected is None:
+ if grading_mode == "custom_only":
return {}
- source, _value = selected
- # Harbor resolves every task-authored placeholder before verifier startup.
- # Override both spellings from the selected host source so a stale alias
- # cannot fail resolution or survive task/step/job environment merging.
- return dict.fromkeys(sorted(_VERIFIER_JUDGE_MODEL_ENV_VARS), f"${{{source}}}")
+ environment = {
+ name: f"${{{name}}}"
+ for name in _VERIFIER_JUDGE_CONTROL_ENV_VARS - _VERIFIER_JUDGE_MODEL_ENV_VARS
+ if provider_env.get(name)
+ }
+ selected = _job_judge_override(provider_env, grading_mode)
+ if selected is not None:
+ source, _value = selected
+ # Harbor resolves every task-authored placeholder before verifier startup.
+ # Override both spellings from the selected host source so a stale alias
+ # cannot fail resolution or survive task/step/job environment merging.
+ environment.update(dict.fromkeys(sorted(_VERIFIER_JUDGE_MODEL_ENV_VARS), f"${{{source}}}"))
+ return environment
def _job_judge_subprocess_env(provider_env: Mapping[str, str], grading_mode: str) -> dict[str, str]:
"""Make both aliases resolvable while Harbor constructs verifier environments."""
- selected = _job_judge_override(provider_env, grading_mode)
- if selected is None:
+ if grading_mode == "custom_only":
return {}
- _source, value = selected
- return dict.fromkeys(_VERIFIER_JUDGE_MODEL_ENV_VARS, value)
+ environment = {
+ name: provider_env[name]
+ for name in _VERIFIER_JUDGE_CONTROL_ENV_VARS - _VERIFIER_JUDGE_MODEL_ENV_VARS
+ if provider_env.get(name)
+ }
+ selected = _job_judge_override(provider_env, grading_mode)
+ if selected is not None:
+ _source, value = selected
+ environment.update(dict.fromkeys(_VERIFIER_JUDGE_MODEL_ENV_VARS, value))
+ return environment
def _agent_credentials(
@@ -1091,7 +2099,7 @@ def _resolve_agent_runtime_plan(
provider_env = {
name: value
for name, value in _provider_environment(provider).items()
- if name not in _VERIFIER_JUDGE_MODEL_ENV_VARS
+ if name not in _VERIFIER_JUDGE_CONTROL_ENV_VARS
}
plans: dict[str, AgentRuntimePlan] = {}
for agent in agents:
@@ -1171,11 +2179,62 @@ def _task_timeout_plan(task_roots: list[Path], timeout_multiplier: float) -> flo
data = tomllib.loads(task_file.read_text(encoding="utf-8"))
except (OSError, tomllib.TOMLDecodeError):
continue
- agent = data.get("agent") if isinstance(data, dict) else None
- value = agent.get("timeout_sec") if isinstance(agent, dict) else None
- if isinstance(value, int | float) and not isinstance(value, bool) and value > 0:
- timeouts.append(float(value))
- return round(max(timeouts) * timeout_multiplier, 3) if timeouts else None
+ for timeout_field, value in _explicit_harbor_timeout_values(data):
+ scaled = _scaled_harbor_timeout(
+ value,
+ timeout_multiplier,
+ task_file=task_file,
+ field=timeout_field,
+ )
+ if timeout_field.endswith("agent.timeout_sec") and scaled > 0:
+ timeouts.append(scaled)
+ return round(max(timeouts), 3) if timeouts else None
+
+
+def _explicit_harbor_timeout_values(
+ value: object,
+) -> Iterator[tuple[str, object]]:
+ """Yield explicit task bases scaled by Harbor's global multiplier."""
+ if not isinstance(value, dict):
+ return
+
+ for section_name, field_name in (
+ ("agent", "timeout_sec"),
+ ("verifier", "timeout_sec"),
+ ("environment", "build_timeout_sec"),
+ ):
+ section = value.get(section_name)
+ if isinstance(section, dict) and field_name in section:
+ yield f"{section_name}.{field_name}", section[field_name]
+
+ steps = value.get("steps")
+ if not isinstance(steps, list):
+ return
+ for index, step in enumerate(steps):
+ if not isinstance(step, dict):
+ continue
+ for section_name in ("agent", "verifier"):
+ section = step.get(section_name)
+ if isinstance(section, dict) and "timeout_sec" in section:
+ yield f"steps.{index}.{section_name}.timeout_sec", section["timeout_sec"]
+
+
+def _scaled_harbor_timeout(
+ value: object,
+ timeout_multiplier: float,
+ *,
+ task_file: Path,
+ field: str,
+) -> float:
+ """Scale one staged timeout without allowing Harbor to receive infinity."""
+ try:
+ base = float(value)
+ scaled = base * timeout_multiplier
+ except (OverflowError, TypeError, ValueError):
+ raise ValueError(f"{task_file}: {field} produces a non-finite Harbor timeout") from None
+ if not math.isfinite(base) or not math.isfinite(scaled):
+ raise ValueError(f"{task_file}: {field} produces a non-finite Harbor timeout")
+ return scaled
def _model_for_agent(
@@ -1227,6 +2286,300 @@ def _nvidia_build_agent_import_path(provider: ProviderConfig, agent: str, env_mo
return None
+def _signal_harbor_process_group(process: subprocess.Popen[bytes], value: signal.Signals) -> None:
+ try:
+ os.killpg(process.pid, value)
+ except ProcessLookupError:
+ return
+ except PermissionError:
+ # macOS can report EPERM instead of ESRCH after the group leader exits.
+ # Suppress only when the original PID is independently gone.
+ if process.poll() is not None:
+ try:
+ os.getpgid(process.pid)
+ except ProcessLookupError:
+ return
+ raise
+
+
+def _windows_system_directory() -> Path:
+ """Return the Windows system directory without consulting ambient environment."""
+ import ctypes
+ from ctypes import wintypes
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ get_system_directory = kernel32.GetSystemDirectoryW
+ get_system_directory.argtypes = [wintypes.LPWSTR, wintypes.UINT]
+ get_system_directory.restype = wintypes.UINT
+ buffer = ctypes.create_unicode_buffer(32768)
+ length = get_system_directory(buffer, len(buffer))
+ if length == 0:
+ raise ctypes.WinError(ctypes.get_last_error())
+ if length >= len(buffer):
+ raise OSError("Windows system directory path exceeded the supported length")
+ system_directory = Path(buffer.value)
+ if not system_directory.is_absolute():
+ raise OSError("Windows system directory was not absolute")
+ return system_directory
+
+
+def _verified_windows_taskkill_path() -> Path:
+ """Resolve a regular, non-reparse taskkill executable inside System32."""
+ system_directory = _windows_system_directory()
+ if not system_directory.is_absolute():
+ raise OSError("Windows system directory was not absolute")
+ candidate = system_directory / "taskkill.exe"
+ try:
+ system_metadata = system_directory.lstat()
+ candidate_metadata = candidate.lstat()
+ resolved_system_directory = system_directory.resolve(strict=True)
+ resolved_candidate = candidate.resolve(strict=True)
+ except (OSError, RuntimeError) as exc:
+ raise OSError("Windows taskkill executable could not be verified") from exc
+
+ reparse_attribute = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
+
+ def is_reparse_point(metadata: os.stat_result) -> bool:
+ return bool(getattr(metadata, "st_file_attributes", 0) & reparse_attribute)
+
+ if (
+ not stat.S_ISDIR(system_metadata.st_mode)
+ or stat.S_ISLNK(system_metadata.st_mode)
+ or is_reparse_point(system_metadata)
+ ):
+ raise OSError("Windows system directory could not be verified")
+ if (
+ not stat.S_ISREG(candidate_metadata.st_mode)
+ or stat.S_ISLNK(candidate_metadata.st_mode)
+ or is_reparse_point(candidate_metadata)
+ ):
+ raise OSError("Windows taskkill executable could not be verified")
+ if resolved_candidate.parent != resolved_system_directory:
+ raise OSError("Windows taskkill executable escaped the system directory")
+ return resolved_candidate
+
+
+def _terminate_harbor_process_tree(process: subprocess.Popen[bytes]) -> None:
+ """Terminate the POSIX process group or Windows task tree and reap Harbor."""
+ if os.name == "posix":
+ # POSIX process groups do not own a descendant that deliberately calls
+ # setsid(2). The host-side Harbor CLI/configuration is therefore a
+ # trusted boundary; untrusted task execution belongs in an isolated
+ # backend rather than SkillEvaluator's experimental local mode.
+ _signal_harbor_process_group(process, signal.SIGTERM)
+ with suppress(subprocess.TimeoutExpired):
+ process.wait(timeout=_HARBOR_RUN_TERMINATE_SECONDS)
+ # Always signal the original group after the grace period. The leader
+ # may already have exited while a descendant retained the output pipe.
+ _signal_harbor_process_group(process, signal.SIGKILL)
+ try:
+ process.wait(timeout=_HARBOR_RUN_REAP_SECONDS)
+ except subprocess.TimeoutExpired as exc:
+ raise RuntimeError("Harbor parent process could not be reaped") from exc
+ return
+
+ taskkill_error: BaseException | None = None
+ taskkill: subprocess.Popen[bytes] | None = None
+ try:
+ taskkill = subprocess.Popen(
+ [str(_verified_windows_taskkill_path()), "/PID", str(process.pid), "/T", "/F"],
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ taskkill.wait(timeout=_HARBOR_RUN_REAP_SECONDS)
+ if taskkill.returncode != 0:
+ taskkill_error = RuntimeError("Windows Harbor process-tree cleanup failed")
+ except subprocess.TimeoutExpired as exc:
+ taskkill_error = exc
+ if taskkill is not None and taskkill.poll() is None:
+ taskkill.kill()
+ try:
+ taskkill.wait(timeout=_HARBOR_RUN_REAP_SECONDS)
+ except subprocess.TimeoutExpired as reap_error:
+ taskkill_error = reap_error
+ except OSError as exc:
+ taskkill_error = exc
+ if process.poll() is None:
+ process.kill()
+ try:
+ process.wait(timeout=_HARBOR_RUN_REAP_SECONDS)
+ except subprocess.TimeoutExpired as exc:
+ raise RuntimeError("Harbor parent process could not be reaped") from exc
+ if taskkill_error is not None:
+ raise RuntimeError("Harbor process-tree cleanup could not be confirmed") from taskkill_error
+
+
+def _run_bounded_harbor_process(
+ command: list[str],
+ *,
+ env: Mapping[str, str],
+ stdin_text: str | None,
+ timeout_seconds: float,
+ max_output_bytes: int,
+ diagnostic_tail_chars: int,
+ secret_values: set[str],
+) -> _BoundedHarborProcessResult:
+ """Run Harbor with bounded merged output and platform cleanup ownership."""
+ if timeout_seconds <= 0:
+ raise ValueError("Harbor run timeout must be positive")
+ if max_output_bytes <= 0:
+ raise ValueError("Harbor output byte limit must be positive")
+ if diagnostic_tail_chars <= 0:
+ raise ValueError("Harbor diagnostic tail limit must be positive")
+
+ creation_flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0
+ process = subprocess.Popen(
+ command,
+ stdin=subprocess.PIPE if stdin_text is not None else subprocess.DEVNULL,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ env=dict(env),
+ start_new_session=os.name == "posix",
+ creationflags=creation_flags,
+ )
+ if process.stdout is None:
+ _terminate_harbor_process_tree(process)
+ raise RuntimeError("Harbor output pipe invariant violated")
+
+ reader_done = threading.Event()
+ output_exceeded = threading.Event()
+ reader_error: list[BaseException] = []
+ stdin_error: list[BaseException] = []
+ output_tail = ""
+ deadline = time.monotonic() + timeout_seconds
+
+ def append_tail(text: str) -> None:
+ nonlocal output_tail
+ if text:
+ output_tail = (output_tail + text)[-diagnostic_tail_chars:]
+
+ def collect_output() -> None:
+ decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+ redactor = StreamingLogRedactor(value for value in secret_values if len(value) >= 4)
+ output_budget = CommandOutputByteBudget(max_output_bytes)
+ reached_eof = False
+ try:
+ output_descriptor = process.stdout.fileno()
+ while True:
+ chunk = os.read(output_descriptor, _HARBOR_RUN_OUTPUT_READ_BYTES)
+ if not chunk:
+ reached_eof = True
+ break
+ remaining = output_budget.limit_bytes - output_budget.consumed_bytes
+ accepted = chunk[: max(remaining, 0)]
+ if accepted:
+ output_budget.consume(accepted)
+ append_tail(redactor.feed(decoder.decode(accepted)))
+ if len(chunk) > remaining:
+ output_exceeded.set()
+ break
+ except BaseException as exc:
+ reader_error.append(exc)
+ finally:
+ if reached_eof:
+ try:
+ append_tail(redactor.feed(decoder.decode(b"", final=True)))
+ append_tail(redactor.finish())
+ except BaseException as exc:
+ reader_error.append(exc)
+ with suppress(OSError):
+ process.stdout.close()
+ reader_done.set()
+
+ def deliver_stdin() -> None:
+ try:
+ if stdin_text is None:
+ return
+ if process.stdin is None:
+ raise RuntimeError("Harbor stdin pipe invariant violated")
+ try:
+ process.stdin.write(stdin_text.encode("utf-8"))
+ process.stdin.flush()
+ except (BrokenPipeError, ConnectionResetError):
+ pass
+ finally:
+ process.stdin.close()
+ except BaseException as exc:
+ stdin_error.append(exc)
+
+ reader = threading.Thread(
+ target=collect_output,
+ name="skillevaluator-harbor-output",
+ daemon=True,
+ )
+ stdin_writer = threading.Thread(
+ target=deliver_stdin,
+ name="skillevaluator-harbor-stdin",
+ daemon=True,
+ )
+ reader_started = False
+ stdin_writer_started = False
+ try:
+ reader.start()
+ reader_started = True
+ stdin_writer.start()
+ stdin_writer_started = True
+ timed_out = False
+ while True:
+ if output_exceeded.is_set() or reader_error or stdin_error:
+ break
+ if reader_done.is_set() and process.poll() is not None:
+ break
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ timed_out = True
+ break
+ reader_done.wait(min(_HARBOR_RUN_POLL_SECONDS, remaining))
+
+ if timed_out or output_exceeded.is_set() or reader_error or stdin_error:
+ _terminate_harbor_process_tree(process)
+ else:
+ process.wait()
+ reader.join(timeout=_HARBOR_RUN_REAP_SECONDS)
+ stdin_writer.join(timeout=_HARBOR_RUN_REAP_SECONDS)
+ if reader.is_alive():
+ raise RuntimeError("Harbor output reader could not be reaped")
+ if stdin_writer.is_alive():
+ raise RuntimeError("Harbor stdin writer could not be reaped")
+ if reader_error:
+ raise RuntimeError("Harbor output collection failed") from reader_error[0]
+ if stdin_error:
+ raise RuntimeError("Harbor stdin delivery failed") from stdin_error[0]
+ if timed_out:
+ detail = f"Harbor run timed out after {timeout_seconds:g} seconds"
+ safe_tail = redact_progress_detail(output_tail, secret_values=secret_values)
+ if safe_tail:
+ detail += f". Last output: {safe_tail[-2000:]}"
+ raise _HarborRunTimeoutError(_redact_harbor_diagnostic(detail, secret_values=secret_values))
+ return _BoundedHarborProcessResult(
+ returncode=int(process.returncode or 0),
+ output_tail=output_tail,
+ output_exceeded=output_exceeded.is_set(),
+ )
+ except BaseException as primary_error:
+ cleanup_error: BaseException | None = None
+ if process.poll() is None or not reader_done.is_set():
+ try:
+ _terminate_harbor_process_tree(process)
+ except BaseException as exc:
+ cleanup_error = exc
+ if not reader_started:
+ with suppress(OSError):
+ process.stdout.close()
+ if not stdin_writer_started and process.stdin is not None:
+ with suppress(OSError):
+ process.stdin.close()
+ if reader_started:
+ reader.join(timeout=_HARBOR_RUN_REAP_SECONDS)
+ if stdin_writer_started:
+ stdin_writer.join(timeout=_HARBOR_RUN_REAP_SECONDS)
+ if cleanup_error is not None:
+ primary_error.add_note(f"Harbor process-tree cleanup also failed: {type(cleanup_error).__name__}")
+ raise primary_error from cleanup_error
+ raise
+
+
def _run_harbor(
*,
dataset: Path,
@@ -1244,10 +2597,16 @@ def _run_harbor(
override_storage_mb: int | None,
agent_import_path: str | None = None,
verifier_env: Mapping[str, str] | None = None,
+ environment_kwargs: Mapping[str, Any] | None = None,
expected_trials: int | None = None,
expected_total_trials: int | None = None,
include_task_names: list[str] | None = None,
) -> tuple[bool, str]:
+ # Preserve the historical exact-value protection for every selected child
+ # value, and additionally protect detached credential URI/proxy userinfo
+ # (including percent-decoded and schemeless proxy components).
+ secret_values = set(run_env.values())
+ secret_values.update(secret_values_from_environment(run_env))
command = build_harbor_run_command(
dataset_path=dataset,
agent=agent,
@@ -1264,30 +2623,46 @@ def _run_harbor(
override_storage_mb=override_storage_mb,
agent_import_path=agent_import_path,
verifier_env=verifier_env,
+ environment_kwargs=environment_kwargs,
)
try:
handoff = _nvidia_build_key_handoff(run_env, env_mode=env_mode)
- result = subprocess.run(
+ result = _run_bounded_harbor_process(
command,
- capture_output=True,
- text=True,
- input=handoff.stdin_text,
env=handoff.subprocess_env,
- timeout=7200,
- check=False,
+ stdin_text=handoff.stdin_text,
+ timeout_seconds=_HARBOR_RUN_TIMEOUT_SECONDS,
+ max_output_bytes=_HARBOR_RUN_OUTPUT_MAX_BYTES,
+ diagnostic_tail_chars=_HARBOR_RUN_DIAGNOSTIC_TAIL_CHARS,
+ secret_values=secret_values,
)
- except (OSError, subprocess.TimeoutExpired) as exc:
- return False, str(exc)
+ except (OSError, RuntimeError, UnicodeError) as exc:
+ detail = _redact_harbor_diagnostic(exc, secret_values=secret_values)
+ if not detail:
+ detail = _redact_harbor_diagnostic(type(exc).__name__, secret_values=secret_values)
+ return False, detail
+ if result.output_exceeded:
+ safe_tail = redact_progress_detail(result.output_tail, secret_values=secret_values)
+ detail = f"harbor run output exceeded the {_HARBOR_RUN_OUTPUT_MAX_BYTES}-byte safety limit"
+ if safe_tail:
+ detail += f". Last output: {safe_tail[-2000:]}"
+ return False, _redact_harbor_diagnostic(detail, secret_values=secret_values)
if result.returncode == 0:
- return _validate_harbor_job_result(
+ validation_ok, validation_detail = _validate_harbor_job_result(
jobs_dir,
job_name,
expected_trials=expected_trials,
expected_total_trials=expected_total_trials,
)
- output = "\n".join(part for part in (result.stderr, result.stdout) if part).strip()
- detail = output[-2000:] or f"harbor run exited {result.returncode}"
- return False, redact_progress_detail(detail, secret_values=set(run_env.values()))
+ if validation_ok:
+ return True, validation_detail
+ return False, _redact_harbor_diagnostic(
+ validation_detail,
+ secret_values=secret_values,
+ )
+ safe_output = redact_progress_detail(result.output_tail, secret_values=secret_values)
+ detail = safe_output[-2000:] or f"harbor run exited {result.returncode}"
+ return False, _redact_harbor_diagnostic(detail, secret_values=secret_values)
def _validate_harbor_job_result(
@@ -1385,6 +2760,63 @@ def _job_root_fingerprint(metadata: os.stat_result) -> tuple[int, int, int, int,
)
+# ``copytree_secure`` stages as ``.{name}.staging-{16 hex}`` in the same
+# directory. Keep the published name below NAME_MAX with room for that suffix.
+_AGGREGATE_TRIAL_NAME_MAX_BYTES = 224
+
+
+def _utc_aware_datetime(value: datetime) -> datetime:
+ """Normalize Harbor timestamps before ordering mixed local/UTC results.
+
+ Harbor 0.22 job summaries can contain naive host-local timestamps while
+ their child trial results use explicit UTC offsets. ``astimezone`` treats
+ a naive value as host-local time, matching the process that wrote the job
+ summary, and gives the aggregate one comparable UTC representation.
+ """
+ return value.astimezone(UTC)
+
+
+def _utf8_prefix(value: str, max_bytes: int) -> str:
+ """Return a UTF-8-safe prefix bounded by encoded byte length."""
+ encoded = value.encode("utf-8")
+ if len(encoded) <= max_bytes:
+ return value
+ return encoded[:max_bytes].decode("utf-8", errors="ignore")
+
+
+def _aggregate_trial_name(
+ job_name: str,
+ child_name: str,
+ *,
+ source_index: int,
+ used_names: set[str],
+) -> str:
+ """Build a deterministic, attempt-readable, filesystem-safe trial name."""
+ raw_name = f"{job_name}__{child_name}"
+ if len(raw_name.encode("utf-8")) > _AGGREGATE_TRIAL_NAME_MAX_BYTES:
+ digest = hashlib.sha256(f"{source_index}\0{raw_name}".encode()).hexdigest()[:16]
+ # stop_on_pass owns the final ``-attemptNNN`` job-name component.
+ # Derive the carried label only from that anchored structure so an
+ # attempt-like task selector or child trial name cannot impersonate it.
+ attempt_match = re.fullmatch(r".+-attempt(0*[1-9][0-9]*)", job_name, flags=re.IGNORECASE)
+ attempt_marker = (
+ f"{TRUNCATED_AGGREGATE_ATTEMPT_PREFIX}{attempt_match.group(1)}" if attempt_match else "__attempt"
+ )
+ suffix = f"{attempt_marker}__{digest}"
+ name = _utf8_prefix(raw_name, _AGGREGATE_TRIAL_NAME_MAX_BYTES - len(suffix.encode())) + suffix
+ else:
+ name = raw_name
+
+ candidate = name
+ collision_index = 2
+ while candidate in used_names:
+ suffix = f"-{collision_index}"
+ candidate = _utf8_prefix(name, _AGGREGATE_TRIAL_NAME_MAX_BYTES - len(suffix.encode())) + suffix
+ collision_index += 1
+ used_names.add(candidate)
+ return candidate
+
+
def _merge_attempt_jobs(job_dirs: list[Path], aggregate_dir: Path) -> None:
"""Merge per-attempt Harbor jobs into the job directory shape collection expects.
@@ -1392,6 +2824,10 @@ def _merge_attempt_jobs(job_dirs: list[Path], aggregate_dir: Path) -> None:
per-attempt Harbor ``result.json`` statistics are combined so the merged
job still satisfies :func:`validate_harbor_job_result`.
"""
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trial.config import TrialConfig
+ from harbor.models.trial.result import TrialResult
+
aggregate_path = Path(os.path.abspath(aggregate_dir)) # noqa: PTH100 -- compare lexical publication roots
source_paths: list[tuple[str, Path, Path, tuple[int, int, int, int, int, int]]] = []
for job_dir in job_dirs:
@@ -1419,7 +2855,7 @@ def _merge_attempt_jobs(job_dirs: list[Path], aggregate_dir: Path) -> None:
aggregate_path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
- prefix=f".{aggregate_path.name}-merge-",
+ prefix=".harbor-merge-",
dir=aggregate_path.parent,
) as private_root_raw:
private_root = Path(private_root_raw)
@@ -1430,7 +2866,7 @@ def _merge_attempt_jobs(job_dirs: list[Path], aggregate_dir: Path) -> None:
snapshots: list[tuple[str, Path]] = []
for index, (job_name, job_path, job_resolved, expected_fingerprint) in enumerate(source_paths):
- snapshot = snapshot_root / f"{index:04d}-{job_name}"
+ snapshot = snapshot_root / f"{index:04d}"
try:
before = job_path.lstat()
except OSError as exc:
@@ -1446,53 +2882,197 @@ def _merge_attempt_jobs(job_dirs: list[Path], aggregate_dir: Path) -> None:
raise ValueError(f"attempt Harbor job root changed during snapshot: {job_path}")
snapshots.append((job_name, snapshot))
+ aggregate_job_id = uuid4()
total_trials = 0
- completed_trials = 0
- errored_trials = 0
- merged_evals: dict[str, dict[str, Any]] = {}
- for job_name, job_dir in snapshots:
- renamed: dict[str, str] = {}
+ aggregate_retries = 0
+ merged_trial_results: list[TrialResult] = []
+ source_started_at: list[datetime] = []
+ source_updated_at: list[datetime] = []
+ used_trial_names: set[str] = set()
+ for source_index, (job_name, job_dir) in enumerate(snapshots):
+ try:
+ source_job_result = json.loads((job_dir / "result.json").read_text(encoding="utf-8"))
+ except (json.JSONDecodeError, OSError):
+ source_job_result = None
+ if isinstance(source_job_result, dict):
+ try:
+ source_job_model = JobResult.model_validate(source_job_result)
+ except Exception:
+ source_job_model = None
+ if source_job_model is not None:
+ source_started_at.append(_utc_aware_datetime(source_job_model.started_at))
+ source_updated_at.append(
+ _utc_aware_datetime(
+ source_job_model.updated_at or source_job_model.finished_at or source_job_model.started_at
+ )
+ )
+ root_trial_results: dict[str, dict[str, Any]] = {}
+ if isinstance(source_job_result, dict) and isinstance(source_job_result.get("trial_results"), list):
+ for candidate in source_job_result["trial_results"]:
+ if isinstance(candidate, dict) and isinstance(candidate.get("trial_name"), str):
+ root_trial_results[candidate["trial_name"]] = candidate
+ if isinstance(source_job_result, dict) and isinstance(source_job_result.get("stats"), dict):
+ source_retries = source_job_result["stats"].get("n_retries")
+ if isinstance(source_retries, int) and not isinstance(source_retries, bool) and source_retries >= 0:
+ aggregate_retries += source_retries
+
+ job_trial_results: list[TrialResult] = []
+ job_trial_slots = 0
for child in sorted(job_dir.iterdir()):
if not child.is_dir():
continue
- dest = staged_aggregate / f"{job_name}__{child.name}"
- suffix = 2
- while dest.exists():
- dest = staged_aggregate / f"{job_name}__{child.name}-{suffix}"
- suffix += 1
+ dest = staged_aggregate / _aggregate_trial_name(
+ job_name,
+ child.name,
+ source_index=source_index,
+ used_names=used_trial_names,
+ )
copytree_secure(child, dest, allowed_root=job_dir)
- renamed[child.name] = dest.name
+
+ result_path = child / "result.json"
+ config_path = child / "config.json"
+ if not result_path.exists() and not config_path.exists():
+ continue
+ job_trial_slots += 1
+ root_trial_payload = root_trial_results.get(child.name)
+ if not result_path.exists() and root_trial_payload is None:
+ try:
+ source_trial_config = TrialConfig.model_validate_json(config_path.read_text(encoding="utf-8"))
+ except Exception as exc:
+ raise ValueError(f"attempt Harbor trial config is invalid: {child}") from exc
+ rewritten_config = source_trial_config.model_dump(mode="json")
+ rewritten_config.update(
+ {
+ "trial_name": dest.name,
+ "trials_dir": str(aggregate_path),
+ "job_id": str(aggregate_job_id),
+ }
+ )
+ try:
+ merged_trial_config = TrialConfig.model_validate(rewritten_config)
+ except Exception as exc:
+ raise ValueError(f"aggregate Harbor trial config is invalid: {dest}") from exc
+ (dest / "config.json").write_text(
+ merged_trial_config.model_dump_json(indent=2),
+ encoding="utf-8",
+ )
+ continue
+ try:
+ trial_payload = json.loads(result_path.read_text(encoding="utf-8"))
+ except (json.JSONDecodeError, OSError):
+ trial_payload = root_trial_payload
+ if not isinstance(trial_payload, dict):
+ raise ValueError(f"attempt Harbor trial result is unreadable: {child}")
+ if trial_payload.get("trial_name") != child.name:
+ raise ValueError(f"attempt Harbor trial result name does not match its directory: {child}")
+ try:
+ source_trial_result = TrialResult.model_validate(trial_payload)
+ except Exception as exc:
+ raise ValueError(f"attempt Harbor trial result is invalid: {child}") from exc
+
+ rewritten = source_trial_result.model_dump(mode="json")
+ rewritten["trial_name"] = dest.name
+ rewritten["trial_uri"] = (aggregate_path / dest.name).as_uri()
+ rewritten_config = rewritten.get("config")
+ if not isinstance(rewritten_config, dict):
+ raise ValueError(f"attempt Harbor trial config is invalid: {child}")
+ rewritten_config.update(
+ {
+ "trial_name": dest.name,
+ "trials_dir": str(aggregate_path),
+ "job_id": str(aggregate_job_id),
+ }
+ )
+ try:
+ merged_trial_result = TrialResult.model_validate(rewritten)
+ except Exception as exc:
+ raise ValueError(f"aggregate Harbor trial result is invalid: {dest}") from exc
+ (dest / "result.json").write_text(
+ merged_trial_result.model_dump_json(indent=2),
+ encoding="utf-8",
+ )
+ (dest / "config.json").write_text(
+ merged_trial_result.config.model_dump_json(indent=2),
+ encoding="utf-8",
+ )
+ job_trial_results.append(merged_trial_result)
+ merged_trial_results.append(merged_trial_result)
stats = _attempt_job_stats(job_dir)
if stats is None:
+ total_trials += job_trial_slots
continue
- job_total, job_completed, job_errored, job_evals = stats
- total_trials += job_total
- completed_trials += job_completed
- errored_trials += job_errored
- for eval_name, (eval_trials, eval_errors, reward_stats) in job_evals.items():
- merged = merged_evals.setdefault(eval_name, {"n_trials": 0, "n_errors": 0, "reward_stats": {}})
- merged["n_trials"] += eval_trials
- merged["n_errors"] += eval_errors
- for metric, buckets in reward_stats.items():
- merged_buckets = merged["reward_stats"].setdefault(metric, {})
- for bucket, trial_names in buckets.items():
- merged_buckets.setdefault(bucket, []).extend(
- renamed.get(name, f"{job_name}__{name}") for name in trial_names
- )
+ job_total, job_completed, _job_errored, _job_evals = stats
+ total_trials += max(job_total, job_trial_slots)
+ if job_completed > len(job_trial_results):
+ raise ValueError(
+ f"attempt Harbor job completed {job_completed} trials but retained "
+ f"{len(job_trial_results)} valid trial results: {job_dir}"
+ )
+ aggregate_config: dict[str, Any] | None = None
+ for _job_name, job_dir in snapshots:
+ try:
+ candidate_config = json.loads((job_dir / "config.json").read_text(encoding="utf-8"))
+ except (json.JSONDecodeError, OSError):
+ continue
+ if isinstance(candidate_config, dict):
+ aggregate_config = candidate_config
+ break
+ if aggregate_config is None:
+ # The aggregate is a retained viewer surface, not a replayable
+ # Harbor invocation. Keep its fallback config valid and explicit
+ # rather than inventing an oracle agent or dataset.
+ aggregate_config = {"agents": [], "datasets": [], "tasks": []}
+ aggregate_config.update(
+ {
+ "job_name": aggregate_path.name,
+ "jobs_dir": str(aggregate_path.parent),
+ "n_attempts": 1,
+ }
+ )
+ (staged_aggregate / "config.json").write_text(
+ json.dumps(aggregate_config, indent=2),
+ encoding="utf-8",
+ )
+
+ trial_finished_at = [
+ _utc_aware_datetime(result.finished_at) for result in merged_trial_results if result.finished_at is not None
+ ]
+ aggregate_updated_at = max([*trial_finished_at, *source_updated_at], default=datetime.now(UTC))
+ aggregate_started_at = min(
+ [
+ *(
+ _utc_aware_datetime(result.started_at)
+ for result in merged_trial_results
+ if result.started_at is not None
+ ),
+ *source_started_at,
+ ],
+ default=aggregate_updated_at,
+ )
+ aggregate_total_trials = max(total_trials, len(merged_trial_results))
+ aggregate_stats = JobStats.from_trial_results(
+ merged_trial_results,
+ n_total_trials=aggregate_total_trials,
+ n_retries=aggregate_retries,
+ )
+ aggregate_finished_at = (
+ max(trial_finished_at, default=aggregate_updated_at)
+ if aggregate_stats.n_completed_trials >= aggregate_total_trials
+ else None
+ )
+ aggregate_result = JobResult(
+ id=aggregate_job_id,
+ started_at=aggregate_started_at,
+ updated_at=aggregate_updated_at,
+ finished_at=aggregate_finished_at,
+ n_total_trials=aggregate_total_trials,
+ stats=aggregate_stats,
+ trial_results=merged_trial_results,
+ )
(staged_aggregate / "result.json").write_text(
- json.dumps(
- {
- "n_total_trials": total_trials,
- "stats": {
- "n_trials": completed_trials,
- "n_errors": errored_trials,
- "evals": merged_evals,
- },
- },
- indent=2,
- ),
+ aggregate_result.model_dump_json(indent=2),
encoding="utf-8",
)
copytree_secure(
@@ -1522,6 +3102,7 @@ def _run_stop_on_pass_variant(
override_storage_mb: int | None,
agent_import_path: str | None = None,
verifier_env: Mapping[str, str] | None = None,
+ environment_kwargs: Mapping[str, Any] | None = None,
) -> list[str]:
"""Run each case one attempt at a time, stopping its attempts on first pass."""
errors: list[str] = []
@@ -1545,6 +3126,7 @@ def _run_stop_on_pass_variant(
override_storage_mb=override_storage_mb,
agent_import_path=agent_import_path,
verifier_env=verifier_env,
+ environment_kwargs=environment_kwargs,
expected_trials=1,
include_task_names=[task_name],
)
@@ -1581,6 +3163,7 @@ def _run_agent_pair(
pass_threshold: float = 0.50,
task_names: list[str] | None = None,
verifier_env: Mapping[str, str] | None = None,
+ environment_kwargs: Mapping[str, Any] | None = None,
) -> list[str]:
jobs = [("with", with_skill)]
if baseline is not None:
@@ -1609,6 +3192,7 @@ def _run_agent_pair(
override_storage_mb=override_storage_mb,
agent_import_path=agent_import_path,
verifier_env=verifier_env,
+ environment_kwargs=environment_kwargs,
)
)
return sequential_errors
@@ -1640,6 +3224,7 @@ def _run_agent_pair(
override_storage_mb=override_storage_mb,
agent_import_path=agent_import_path,
verifier_env=verifier_env,
+ environment_kwargs=environment_kwargs,
expected_trials=expected_trials,
): variant
for (variant, dataset), condition_concurrency in zip(jobs, job_concurrency, strict=True)
@@ -1796,6 +3381,7 @@ def _run_harbor_eval_impl(
agent_runtime_preflight: bool | None = None,
env_mode: str = DEFAULT_ENV_MODE,
env_mode_source: str = "CLI",
+ environment_kwargs: Mapping[str, Any] | None = None,
timeout_multiplier: float | None = None,
override_cpus: int | None = None,
override_memory_mb: int | None = None,
@@ -1869,6 +3455,16 @@ def _run_harbor_eval_impl(
# ``reuse``/``rebuild`` opt into the shared pre-built eval base image.
base_image_mode = harbor_config.get("base_image_mode", "disabled")
task_source = harbor_config.get("task_source", "auto")
+ try:
+ effective_environment_kwargs = validate_environment_kwargs(
+ dict(environment_kwargs or {}),
+ env_mode=env_mode,
+ )
+ except (RecursionError, TypeError, ValueError) as exc:
+ detail = f"Invalid --environment-kwarg: {exc}"
+ reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=detail))
+ return {"error": [detail]}
+ environment_kwarg_sources = dict.fromkeys(effective_environment_kwargs, "CLI")
if not isinstance(n_attempts, int) or n_attempts < 1:
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid attempt count"))
@@ -1882,15 +3478,29 @@ def _run_harbor_eval_impl(
if not isinstance(max_agents, int) or max_agents < 1:
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid agent concurrency"))
return {"error": ["max_agents must be >= 1"]}
- if not isinstance(pass_threshold, (int, float)) or not 0 <= float(pass_threshold) <= 1:
+ try:
+ timeout_multiplier = _validated_timeout_multiplier(timeout_multiplier)
+ except ValueError as exc:
+ reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid timeout multiplier"))
+ return {"error": [str(exc)]}
+ try:
+ pass_threshold = _validated_pass_threshold(pass_threshold)
+ except ValueError as exc:
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid pass threshold"))
- return {"error": ["pass_threshold must be between 0.0 and 1.0"]}
+ return {"error": [str(exc)]}
if grading_mode not in {"default", "default_plus_custom", "custom_only"}:
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid grading mode"))
return {"error": ["grading.mode must be default, default_plus_custom, or custom_only"]}
if workspace_mode not in {"isolated", "group"}:
reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid workspace mode"))
return {"error": ["skill_workspace.mode must be isolated or group"]}
+ if not skip_baseline and harbor_config.get("pre_agent_setup"):
+ detail = (
+ "harbor.pre_agent_setup/setup_commands cannot run in a paired evaluation; "
+ "use --skip-baseline for a with-skill-only run"
+ )
+ reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=detail))
+ return {"error": [detail]}
reporter.emit(ProgressEvent(stage="configuration", state="ready", detail="evaluation config validated"))
reporter.emit(ProgressEvent(stage="model-resolution", state="running"))
@@ -1929,8 +3539,36 @@ def _run_harbor_eval_impl(
)
)
+ prerequisite_subprocess_env: dict[str, str] | None = None
+ if env_mode in {"ack", "opensandbox"} and agents:
+ preflight_agent = agents[0]
+ runtime_provider_env = {
+ name: value for name, value in provider_env.items() if name not in _VERIFIER_JUDGE_CONTROL_ENV_VARS
+ }
+ prerequisite_subprocess_env = _harbor_subprocess_environment(
+ env_mode=env_mode,
+ provider=provider,
+ configured_runtime_env=configured_runtime_env,
+ provider_env=runtime_provider_env,
+ agent=preflight_agent,
+ agent_model=model_resolution[preflight_agent]["model"],
+ )
+ prerequisite_subprocess_env.update(
+ _agent_credentials(
+ provider=provider,
+ agent=preflight_agent,
+ env_mode=env_mode,
+ )
+ )
+ prerequisite_subprocess_env.update(_job_judge_subprocess_env(provider_env, grading_mode))
+
reporter.emit(ProgressEvent(stage="environment-preflight", state="running", detail=env_mode))
- prereq_errors = _check_prerequisites(env_mode=env_mode, agents=agents)
+ prereq_errors = _check_prerequisites(
+ env_mode=env_mode,
+ agents=agents,
+ environment_kwargs=effective_environment_kwargs,
+ subprocess_env=prerequisite_subprocess_env,
+ )
if prereq_errors:
reporter.emit(ProgressEvent(stage="environment-preflight", state="failed", detail="; ".join(prereq_errors)))
return {"error": prereq_errors}
@@ -2133,6 +3771,10 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None:
"config_file": str(config_path.relative_to(evaluator_skill_path)) if config_path else "none",
"harbor": {
"environment": {"value": env_mode, "source": env_mode_source},
+ "environment_kwargs": {
+ "keys": sorted(effective_environment_kwargs),
+ "sources": environment_kwarg_sources,
+ },
"n_attempts": n_attempts,
"stop_on_pass": bool(stop_on_pass),
"n_concurrent": n_concurrent,
@@ -2151,7 +3793,12 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None:
"agents": model_resolution,
}
verifier_env = {**configured_runtime_env, **provider_env}
- staged_verifier_env = {name: f"${{{name}}}" for name in verifier_env if name not in _VERIFIER_JUDGE_MODEL_ENV_VARS}
+ staged_verifier_env = {
+ name: f"${{{name}}}"
+ for name in verifier_env
+ if name not in _VERIFIER_JUDGE_MODEL_ENV_VARS
+ and not (grading_mode == "custom_only" and name in _VERIFIER_JUDGE_CONTROL_ENV_VARS)
+ }
job_judge_verifier_env = _job_judge_verifier_env(provider_env, grading_mode)
job_judge_subprocess_env = _job_judge_subprocess_env(provider_env, grading_mode)
@@ -2212,11 +3859,17 @@ def _emit_run_finished(state: str, detail: str, *, include_artifacts: bool = Tru
def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
"""Retain redacted probe provenance for failures after run reservation."""
+ published_errors, error_total = _published_execution_errors(errors)
failed_result: dict[str, Any] = {
"skill_name": skill_path.name,
"execution_status": "failed",
- "execution_errors": errors,
- "error": errors,
+ "execution_errors": published_errors,
+ "execution_error_details_total": error_total,
+ "execution_error_details_shown": len(published_errors),
+ "execution_error_details_truncated": len(published_errors) < error_total,
+ # Keep the legacy list-shaped failure alias without serializing the
+ # complete diagnostic sample a second time.
+ "error": published_errors[:1],
"run_id": run_id,
"run_dir": str(run_dir),
"harbor_jobs_dir": str(jobs_dir),
@@ -2230,7 +3883,7 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
run_dir / "run_config.json",
json.dumps(run_config, indent=2).encode("utf-8"),
)
- write_output_file_atomically(result_path, json.dumps(failed_result, indent=2).encode("utf-8"))
+ _write_final_result(result_path, failed_result)
return failed_result
reservation_identity: tuple[int, int] | None = None
@@ -2278,7 +3931,8 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
)
)
agent_task_dirs: dict[str, tuple[Path, Path | None]] = {}
- expected_task_names: list[str] | None = None
+ expected_task_selectors: list[str] | None = None
+ expected_case_id_by_task_selector: dict[str, str] | None = None
reporter.emit(
ProgressEvent(
stage="with-skill-tasks",
@@ -2311,11 +3965,17 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
agent_workdir=harbor_config.get("agent_workdir"),
evaluator_skill_path=evaluator_skill_path,
)
- task_names = [task.name for task in task_paths]
- if expected_task_names is None:
- expected_task_names = task_names
- elif task_names != expected_task_names:
- raise ValueError(f"Generated task cases differ for agent {agent}")
+ task_selectors = validate_case_ids(task.name for task in task_paths)
+ logical_case_ids = validate_case_ids(_native_entry_id(task) for task in task_paths)
+ case_id_by_task_selector = dict(zip(task_selectors, logical_case_ids, strict=True))
+ if expected_task_selectors is None:
+ expected_task_selectors = task_selectors
+ expected_case_id_by_task_selector = case_id_by_task_selector
+ elif (
+ task_selectors != expected_task_selectors
+ or case_id_by_task_selector != expected_case_id_by_task_selector
+ ):
+ raise ValueError(f"Generated task identities differ for agent {agent}")
agent_task_dirs[agent] = (with_dir, without_dir)
reporter.emit(ProgressEvent(stage="with-skill-tasks", state="ready", detail="task inputs staged"))
if not skip_baseline:
@@ -2332,7 +3992,7 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
for agent in agents:
without_dir = agent_task_dirs[agent][1]
if without_dir is not None:
- emitter(
+ baseline_task_paths = emitter(
skill_path,
without_dir,
with_skill=False,
@@ -2352,6 +4012,16 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
evaluator_skill_path=evaluator_skill_path,
_baseline_alias_validation=baseline_alias_validation,
)
+ baseline_task_selectors = validate_case_ids(task.name for task in baseline_task_paths)
+ baseline_logical_case_ids = validate_case_ids(_native_entry_id(task) for task in baseline_task_paths)
+ baseline_case_id_by_task_selector = dict(
+ zip(baseline_task_selectors, baseline_logical_case_ids, strict=True)
+ )
+ if (
+ baseline_task_selectors != expected_task_selectors
+ or baseline_case_id_by_task_selector != expected_case_id_by_task_selector
+ ):
+ raise ValueError(f"Baseline task identities differ for agent {agent}")
if not skip_baseline:
reporter.emit(ProgressEvent(stage="baseline-tasks", state="ready", detail="baseline inputs staged"))
else:
@@ -2360,15 +4030,25 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
reporter.emit(ProgressEvent(stage=staging_failure_stage, state="failed", detail=str(exc)))
return _persist_pre_execution_failure([str(exc)])
- task_names = expected_task_names or []
- expected_trials = len(task_names) * n_attempts
+ task_selectors = expected_task_selectors or []
+ case_id_by_task_selector = expected_case_id_by_task_selector or {}
+ case_ids = [case_id_by_task_selector[selector] for selector in task_selectors]
+ try:
+ dataset_truth = _persist_dataset_truth(run_dir, fallback_task_ids=case_ids)
+ except DatasetSnapshotContractError as exc:
+ return _persist_pre_execution_failure([str(exc)])
+ expected_trials = len(task_selectors) * n_attempts
variants = 1 if skip_baseline else 2
matrix_trials = expected_trials * len(agents) * variants
preflight_trials = len(agents) if agent_runtime_preflight else 0
- task_timeout_seconds = _task_timeout_plan(
- [paths[0] for paths in agent_task_dirs.values()],
- float(timeout_multiplier),
- )
+ try:
+ task_timeout_seconds = _task_timeout_plan(
+ [paths[0] for paths in agent_task_dirs.values()],
+ float(timeout_multiplier),
+ )
+ except ValueError as exc:
+ reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid staged timeout"))
+ return _persist_pre_execution_failure([str(exc)])
reporter.start(
Tier3RunPlan(
skill_name=skill_path.name,
@@ -2376,8 +4056,8 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
agents=tuple(agents),
agent_models=tuple((agent, model_resolution[agent]["model"]) for agent in agents),
provider=provider.provider,
- task_count=len(task_names),
- case_count=len(task_names),
+ task_count=len(task_selectors),
+ case_count=len(case_ids),
attempts=n_attempts,
baseline=not skip_baseline,
concurrency=n_concurrent,
@@ -2420,6 +4100,7 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]:
override_memory_mb=override_memory_mb,
override_storage_mb=override_storage_mb,
agent_import_path=nvidia_build_agent_import_paths.get(agent),
+ environment_kwargs=effective_environment_kwargs,
)
if not preflight.ok:
preflight_errors.append(f"{agent} runtime preflight failed: {preflight.detail}")
@@ -2462,8 +4143,9 @@ def _execute_agent(agent: str) -> list[str]:
expected_trials=expected_trials,
stop_on_pass=bool(stop_on_pass),
pass_threshold=float(pass_threshold),
- task_names=task_names,
+ task_names=task_selectors,
verifier_env=job_judge_verifier_env,
+ environment_kwargs=effective_environment_kwargs,
)
active_agents: set[str] = set()
@@ -2530,8 +4212,9 @@ def _emit_started_agents() -> None:
n_attempts=n_attempts,
pass_threshold=float(pass_threshold),
stop_on_pass=bool(stop_on_pass),
- expected_cases=len(task_names),
- expected_case_ids=task_names,
+ expected_cases=len(case_ids),
+ expected_case_ids=case_ids,
+ case_id_by_task_selector=case_id_by_task_selector,
# Early-stopped cases legitimately use fewer trials than the
# n_attempts maximum; per-case coverage is validated instead.
expected_trials=None if stop_on_pass else expected_trials,
@@ -2544,7 +4227,6 @@ def _emit_started_agents() -> None:
_emit_run_finished("failed", "result collection failed")
raise
reporter.emit(ProgressEvent(stage="collection", state="complete", detail="Harbor results collected"))
- dataset_truth = _persist_dataset_truth(run_dir, fallback_task_ids=task_names)
results.update(
{
"skill_name": skill_path.name,
@@ -2555,7 +4237,7 @@ def _emit_started_agents() -> None:
"harbor_jobs_retained": keep_harbor_jobs,
"evaluated_at": datetime.now(UTC).isoformat(),
"evaluator_version": dataset_truth["evaluator_version"],
- "dataset_snapshot": dataset_truth,
+ "dataset_snapshot": dataset_snapshot_manifest(dataset_truth),
"dataset_snapshot_path": str(run_dir / "dataset_snapshot.json"),
"dataset_summary": dataset_truth["dataset_summary"],
"dataset_digest": dataset_truth["dataset_digest"],
@@ -2575,12 +4257,7 @@ def _emit_started_agents() -> None:
result=results,
)
if errors:
- execution_errors = list(
- dict.fromkeys([*(str(error) for error in results.get("execution_errors", [])), *errors])
- )
- results["execution_status"] = "failed"
- results["execution_errors"] = execution_errors
- results["error"] = execution_errors
+ _merge_launch_execution_errors(results, errors)
reporter.emit(ProgressEvent(stage="report", state="running"))
try:
(run_dir / "run_config.json").write_text(json.dumps(run_config, indent=2), encoding="utf-8")
@@ -2612,7 +4289,7 @@ def _emit_started_agents() -> None:
results["duration_seconds"] = round(time.monotonic() - started_at, 3)
try:
- write_output_file_atomically(result_path, json.dumps(results, indent=2).encode("utf-8"))
+ _write_final_result(result_path, results)
except Exception:
reporter.emit(ProgressEvent(stage="report", state="failed", detail="result write failed"))
_emit_run_finished("failed", "report artifacts could not be written")
@@ -2672,7 +4349,7 @@ def _finalize_harbor_artifacts(
result_path_value = result.get("result_path")
result_path = Path(str(result_path_value)) if result_path_value else run_dir / "result.json"
if result_path.is_file():
- write_output_file_atomically(result_path, json.dumps(result, indent=2).encode("utf-8"))
+ _write_final_result(result_path, result)
run_config = result.get("run_config")
run_config_path = run_dir / "run_config.json"
if isinstance(run_config, dict) and run_config_path.is_file():
diff --git a/src/skillevaluator/tier3/harbor/runtime_preflight.py b/src/skillevaluator/tier3/harbor/runtime_preflight.py
index d7ce0d55..dea27d54 100644
--- a/src/skillevaluator/tier3/harbor/runtime_preflight.py
+++ b/src/skillevaluator/tier3/harbor/runtime_preflight.py
@@ -19,7 +19,7 @@
from queue import Empty, Queue
from threading import BoundedSemaphore, Thread
from time import monotonic
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from urllib.parse import urlsplit
from urllib.request import getproxies
@@ -186,6 +186,9 @@ class CredentialProbeDisposition(StrEnum):
"AWS_ENDPOINT_URL",
"AWS_ENDPOINT_URL_BEDROCK",
"AWS_ENDPOINT_URL_BEDROCK_RUNTIME",
+ "AWS_ENDPOINT_URL_SIGNIN",
+ "AWS_ENDPOINT_URL_SSO",
+ "AWS_ENDPOINT_URL_SSO_OIDC",
"AWS_ENDPOINT_URL_STS",
)
_BEDROCK_RUNTIME_SERVICE_MODELS = ("bedrock", "bedrock-runtime")
@@ -848,7 +851,7 @@ def validate_harbor_agent_only_job_result(
) -> tuple[bool, str]:
"""Validate a verification-disabled Harbor job and its agent result.
- Harbor 0.13.2 records an agent-only trial as completed at the job level,
+ Harbor 0.22 records an agent-only trial as completed at the job level,
but intentionally leaves its evaluation trial and reward counts at zero.
The per-trial result is therefore the proof that the agent actually ran.
"""
@@ -1359,6 +1362,7 @@ def run_agent_runtime_preflight(
override_memory_mb: int | None = None,
override_storage_mb: int | None = None,
agent_import_path: str | None = None,
+ environment_kwargs: Mapping[str, Any] | None = None,
) -> PreflightResult:
"""Start one real agent task and stop before the full A/B matrix."""
task_name = _first_task_name(dataset)
@@ -1382,6 +1386,7 @@ def run_agent_runtime_preflight(
override_memory_mb=override_memory_mb,
override_storage_mb=override_storage_mb,
agent_import_path=agent_import_path,
+ environment_kwargs=environment_kwargs,
)
try:
handoff = _nvidia_build_key_handoff(run_env, env_mode=env_mode)
diff --git a/src/skillevaluator/tier3/harbor/secure_docker_environment.py b/src/skillevaluator/tier3/harbor/secure_docker_environment.py
index 4218bd65..c010eb16 100644
--- a/src/skillevaluator/tier3/harbor/secure_docker_environment.py
+++ b/src/skillevaluator/tier3/harbor/secure_docker_environment.py
@@ -14,29 +14,63 @@
from __future__ import annotations
import asyncio
+import codecs
import contextlib
+import contextvars
+import math
import os
import re
import shlex
+import shutil
import signal
+import stat
+import tarfile
+import tempfile
+import unicodedata
import uuid
-from collections.abc import Mapping
-from pathlib import Path
-from typing import Any
+from collections import deque
+from collections.abc import AsyncIterator, Iterable, Iterator, Mapping
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Any, BinaryIO
-from harbor.environments.base import ExecResult
+import yaml
+from harbor.environments.base import (
+ MAIN_SERVICE_NAME,
+ ExecResult,
+ OutputCallback,
+ OutputStream,
+ ServiceOperationsUnsupportedError,
+)
from harbor.environments.docker.docker import DockerEnvironment, _sanitize_docker_compose_project_name
+from skillevaluator.tier3.harbor.progress import secret_values_from_environment
+from skillevaluator.tier3.harbor.secure_copy import (
+ _absolute_lexical,
+ copy_file_secure,
+ copytree_secure,
+)
from skillevaluator.tier3.harbor.sensitive_stdin import (
+ NVIDIA_BUILD_KEY_STDIN_ENV,
NVIDIA_BUILD_STDIN_SENTINEL,
read_nvidia_build_key_from_stdin,
)
+from skillevaluator.tier3.harbor.stream_redaction import CommandOutputByteBudget
+from skillevaluator.utils.secure_fs import stat_is_link_or_reparse
SECURE_DOCKER_ENV_IMPORT_PATH = (
"skillevaluator.tier3.harbor.secure_docker_environment:SkillEvaluatorSecureDockerEnvironment"
)
_ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_ENV_NAME_PREFIX_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+_COMPOSE_SERVICE_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
+_SENSITIVE_ENV_NAME_RE = re.compile(
+ r"(?:^|_)(?:API_?KEY|ACCESS_?KEY|PRIVATE_?KEY|KEY|PAT|TOKEN|SECRET|PASS(?:WORD)?|"
+ r"CREDENTIALS?|AUTH(?:ORIZATION)?|BEARER|COOKIE|SESSION|CERT(?:IFICATE)?|DSN|"
+ r"CONNECTION(?:_STRING)?|(?:PRE)?SIGNED_?URL|SAS_?URL|CREDENTIAL_?URL|DATABASE_?URL)(?:_|$)",
+ re.IGNORECASE,
+)
_NVIDIA_BUILD_FILE_SENTINEL = "skillevaluator-file-backed-nvidia-key"
_NVIDIA_BUILD_KEY_FILE_ENV = "SKILLEVALUATOR_NVIDIA_API_KEY_FILE"
# Match llm_judge / local_environment: short env values like "1" must not
@@ -45,7 +79,406 @@
_COMPOSE_TERMINATE_SECONDS = 5.0
_COMPOSE_KILL_SECONDS = 5.0
_COMPOSE_CANCEL_SECONDS = 0.1
-_MAIN_CONTAINER_STOP_TIMEOUT_SECONDS = 8
+_RAW_DOCKER_COMMAND_TIMEOUT_SECONDS = 3.0
+_RAW_LIFECYCLE_TOTAL_TIMEOUT_SECONDS = 30.0
+_SIDECAR_ENV_CARRIER_PREFIX = "SKILLEVALUATOR_SIDECAR_ENV_"
+_MAX_COMPOSE_MODEL_FILES = 64
+_MAX_COMPOSE_MODEL_BYTES = 8 * 1024 * 1024
+_MAX_COMPOSE_MODEL_NODES = 100_000
+_MAX_COMPOSE_MODEL_DEPTH = 128
+_WINDOWS_TRANSFER_IDLE_TIMEOUT_SECONDS = 300.0
+_WINDOWS_TRANSFER_TOTAL_TIMEOUT_SECONDS = 1800.0
+_WINDOWS_TRANSFER_POLL_SECONDS = 0.05
+_WINDOWS_TRANSFER_STDERR_MAX_BYTES = 1024 * 1024
+_WINDOWS_TAR_MAX_MEMBERS = 100_000
+_WINDOWS_TAR_MAX_FILESYSTEM_ENTRIES = 100_000
+_WINDOWS_TAR_MAX_PATH_BYTES = 8 * 1024 * 1024
+_WINDOWS_TAR_MAX_PATH_COMPONENTS = 1_000_000
+_WINDOWS_TAR_MAX_DEPTH = 128
+_WINDOWS_TAR_MAX_LEADING_CURRENT_COMPONENTS = 8
+_WINDOWS_TAR_MAX_MEMBER_PATH_BYTES = 128 * 1024
+_WINDOWS_TAR_MAX_EXTENSION_BYTES = 1024 * 1024
+_WINDOWS_TAR_MAX_TOTAL_EXTENSION_BYTES = 8 * 1024 * 1024
+_WINDOWS_TAR_MAX_EXTENSION_CHAIN = 32
+_WINDOWS_TAR_MAX_PAX_RECORDS_PER_HEADER = 4096
+_WINDOWS_ARTIFACT_DISK_RESERVE_BYTES = 512 * 1024 * 1024
+_WINDOWS_ARTIFACT_ENTRY_DISK_BYTES = 64 * 1024
+_WINDOWS_ARTIFACT_INODE_RESERVE = 1024
+_TAR_BLOCK_BYTES = 512
+_WINDOWS_RESERVED_NAME_RE = re.compile(
+ r"(?:con|prn|aux|nul|conin\$|conout\$|com[1-9¹²³]|lpt[1-9¹²³])",
+ re.IGNORECASE,
+)
+_REDACTION_LABEL = "[REDACTED]"
+_REDACTION_SENTINEL_CANDIDATES = ("␟", "␞", "␝", "␜", "")
+
+
+@dataclass(frozen=True, slots=True, repr=False)
+class _SecureHandoffScope:
+ environment_names: frozenset[str]
+ secret_values: frozenset[str]
+
+
+@dataclass(slots=True, eq=False)
+class _SidecarOperation:
+ environment_identity: int
+ service: str
+ compose_model_environment: dict[str, str]
+ active: bool = True
+
+
+@dataclass(frozen=True, slots=True)
+class _RawContainerState:
+ identity: str
+ project: str
+ service: str
+ container_number: int
+ running: bool
+ paused: bool
+ restarting: bool
+ status: str
+ health_status: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class _RawServiceSnapshot:
+ all_identities: tuple[str, ...]
+ running_identities: tuple[str, ...]
+
+
+_SECURE_HANDOFF_SCOPES: contextvars.ContextVar[tuple[_SecureHandoffScope, ...]] = contextvars.ContextVar(
+ "skillevaluator_secure_docker_handoff_scopes",
+ default=(),
+)
+_SIDECAR_EXEC_OPERATIONS: contextvars.ContextVar[tuple[_SidecarOperation, ...]] = contextvars.ContextVar(
+ "skillevaluator_sidecar_exec_operations",
+ default=(),
+)
+_RAW_LIFECYCLE_DEADLINE: contextvars.ContextVar[float | None] = contextvars.ContextVar(
+ "skillevaluator_raw_docker_lifecycle_deadline",
+ default=None,
+)
+
+
+class _ComposeCommandTimeout(Exception):
+ """Internal marker that cannot collide with callback exception types."""
+
+
+class _WindowsTransferDiskBudgetExceeded(Exception):
+ """Internal marker for untrusted transfer spool exhaustion."""
+
+
+class _WindowsTransferTotalTimeout(Exception):
+ """Internal marker for a transfer that keeps making progress forever."""
+
+
+class _ComposeModelLoader(yaml.SafeLoader):
+ """Safe YAML loader that preserves Compose override-tagged values."""
+
+
+def _construct_compose_override_value(
+ loader: _ComposeModelLoader,
+ node: yaml.Node,
+) -> object:
+ if isinstance(node, yaml.ScalarNode):
+ return loader.construct_scalar(node)
+ if isinstance(node, yaml.SequenceNode):
+ return loader.construct_sequence(node)
+ if isinstance(node, yaml.MappingNode):
+ return loader.construct_mapping(node)
+ raise yaml.constructor.ConstructorError(
+ None,
+ None,
+ "unsupported Docker Compose override value",
+ node.start_mark,
+ )
+
+
+for _compose_override_tag in ("!reset", "!override"):
+ _ComposeModelLoader.add_constructor(
+ _compose_override_tag,
+ _construct_compose_override_value,
+ )
+
+
+class _SecretTrieNode:
+ __slots__ = ("children", "failure", "max_terminal_length", "terminal_length")
+
+ def __init__(self) -> None:
+ self.children: dict[str, _SecretTrieNode] = {}
+ self.failure = self
+ self.max_terminal_length = 0
+ self.terminal_length = 0
+
+
+def _eligible_secret_values(
+ secret_values: Iterable[str],
+ *,
+ include_short: bool = False,
+) -> list[str]:
+ return sorted(
+ {value for value in secret_values if value and (include_short or len(value) >= _MIN_EXACT_SECRET_LENGTH)}
+ )
+
+
+def _sensitive_environment_values(environment: Mapping[str, str]) -> set[str]:
+ """Return exact values whose component-aware names mark them sensitive."""
+ return {value for name, value in environment.items() if value and _SENSITIVE_ENV_NAME_RE.search(name)}
+
+
+def _credential_uri_environment_values(environment: Mapping[str, str]) -> set[str]:
+ """Return operational URI values containing authority user information."""
+ candidates = {
+ name: value
+ for name, value in environment.items()
+ if value and (name.upper().endswith("_PROXY") or "://" in value)
+ }
+ return secret_values_from_environment(candidates)
+
+
+def _compose_client_credential_values(environment: Mapping[str, str]) -> set[str]:
+ """Return credential-bearing values used by the Docker Compose client."""
+ protected = _credential_uri_environment_values(environment)
+ protected.update(value for name, value in environment.items() if value and name.upper() == "DOCKER_AUTH_CONFIG")
+ return protected
+
+
+def _value_contains_protected_value(value: str, protected_value: str) -> bool:
+ """Match protected values inside structurally retained Compose values."""
+ return protected_value in value
+
+
+def _collision_safe_redaction_marker(
+ secret_values: Iterable[str],
+ *,
+ include_short: bool = False,
+) -> str:
+ """Build a marker that cannot contain or join into an eligible secret."""
+ secrets = _eligible_secret_values(secret_values, include_short=include_short)
+ if not secrets:
+ return _REDACTION_LABEL
+
+ sentinel: str | None = None
+ for candidate in _REDACTION_SENTINEL_CANDIDATES:
+ if all(candidate not in secret for secret in secrets):
+ sentinel = candidate
+ break
+
+ if sentinel is None:
+ used_characters: set[str] = set()
+ for secret in secrets:
+ used_characters.update(secret)
+
+ # Private-use scalars have no standardized control semantics. If they
+ # are all occupied, accept only Unicode letters, numbers, punctuation,
+ # or symbols; controls, separators, combining marks, and surrogates
+ # are unsafe as callback and diagnostic boundaries.
+ private_use_ranges = (
+ range(0xE000, 0xF900),
+ range(0xF0000, 0xFFFFE),
+ range(0x100000, 0x10FFFE),
+ )
+ for candidate_range in private_use_ranges:
+ for codepoint in candidate_range:
+ candidate = chr(codepoint)
+ if candidate not in used_characters:
+ sentinel = candidate
+ break
+ if sentinel is not None:
+ break
+
+ if sentinel is None:
+ scalar_ranges = (range(1, 0xD800), range(0xE000, 0x110000))
+ for scalar_range in scalar_ranges:
+ for codepoint in scalar_range:
+ candidate = chr(codepoint)
+ if candidate not in used_characters and unicodedata.category(candidate)[0] in "LNPS":
+ sentinel = candidate
+ break
+ if sentinel is not None:
+ break
+
+ if sentinel is None:
+ # No marker can satisfy the absent-character invariant. Fail without
+ # rendering any secret value; callers construct this before spawning.
+ raise RuntimeError("Could not construct a collision-safe redaction marker")
+
+ minimum_secret_length = min(map(len, secrets))
+ if minimum_secret_length == 1:
+ return sentinel
+ chunk_length = minimum_secret_length - 1
+ # Sentinel boundaries prevent a secret from bridging raw text and the
+ # marker. Splitting the readable label keeps every sentinel-free run below
+ # the shortest eligible secret length, so no secret can live in the marker.
+ label_chunks = [
+ _REDACTION_LABEL[index : index + chunk_length] for index in range(0, len(_REDACTION_LABEL), chunk_length)
+ ]
+ return sentinel + sentinel.join(label_chunks) + sentinel
+
+
+class _StreamingSecretRedactor:
+ """Redact the union of secret-match spans using Aho-Corasick.
+
+ Collapsing every connected covered span deliberately redacts more than
+ leftmost-longest replacement when secrets overlap. Each automaton state
+ stores only its longest terminal suffix, because that interval covers all
+ shorter matches ending at the same position.
+ """
+
+ def __init__(
+ self,
+ secret_values: Iterable[str],
+ *,
+ _replacement: str | None = None,
+ _track_transitions: bool = False,
+ _include_short: bool = False,
+ ) -> None:
+ secrets = _eligible_secret_values(
+ secret_values,
+ include_short=_include_short,
+ )
+ self._root = _SecretTrieNode()
+ for secret in secrets:
+ node = self._root
+ for character in secret:
+ child = node.children.get(character)
+ if child is None:
+ child = _SecretTrieNode()
+ node.children[character] = child
+ node = child
+ node.terminal_length = len(secret)
+
+ failure_queue = deque(self._root.children.values())
+ for child in failure_queue:
+ child.failure = self._root
+ child.max_terminal_length = child.terminal_length
+ while failure_queue:
+ node = failure_queue.popleft()
+ for character, child in node.children.items():
+ failure = node.failure
+ while failure is not self._root and character not in failure.children:
+ failure = failure.failure
+ child.failure = failure.children.get(character, self._root)
+ child.max_terminal_length = max(
+ child.terminal_length,
+ child.failure.max_terminal_length,
+ )
+ failure_queue.append(child)
+
+ self._has_secrets = bool(secrets)
+ self._max_secret_length = max(map(len, secrets), default=0)
+ self._state = self._root
+ self._pending: deque[str] = deque()
+ self._pending_start = 0
+ self._processed = 0
+ self._coverage: deque[tuple[int, int]] = deque()
+ self._redaction_open = False
+ self._replacement = (
+ _collision_safe_redaction_marker(secrets, include_short=True) if _replacement is None else _replacement
+ )
+ self._track_transitions = _track_transitions
+ self._match_transition_count = 0
+ self._match_work_count = 0
+
+ @property
+ def match_transition_count(self) -> int:
+ return self._match_transition_count
+
+ @property
+ def match_work_count(self) -> int:
+ """Return instrumented scan, coverage, and commit operations."""
+ return self._match_work_count
+
+ def _record_work(self) -> None:
+ if self._track_transitions:
+ self._match_work_count += 1
+
+ def _add_coverage(self, start: int, end: int) -> None:
+ merged_start = start
+ while True:
+ self._record_work()
+ if not self._coverage or self._coverage[-1][1] < merged_start:
+ break
+ previous_start, _previous_end = self._coverage.pop()
+ merged_start = min(merged_start, previous_start)
+ self._record_work()
+ self._coverage.append((merged_start, end))
+ self._record_work()
+
+ def _advance(self, character: str) -> None:
+ while True:
+ if self._track_transitions:
+ self._match_transition_count += 1
+ self._match_work_count += 1
+ child = self._state.children.get(character)
+ if child is not None:
+ self._state = child
+ break
+ if self._state is self._root:
+ break
+ self._state = self._state.failure
+
+ match_end = self._processed + 1
+ match_length = self._state.max_terminal_length
+ self._record_work()
+ if match_length:
+ self._add_coverage(match_end - match_length, match_end)
+ self._processed = match_end
+
+ def _position_is_covered(self, position: int) -> bool:
+ while True:
+ self._record_work()
+ if not self._coverage or self._coverage[0][1] > position:
+ break
+ self._coverage.popleft()
+ self._record_work()
+ self._record_work()
+ return bool(self._coverage and self._coverage[0][0] <= position < self._coverage[0][1])
+
+ def _drain(self, *, final: bool) -> str:
+ if final:
+ safe_end = self._processed
+ else:
+ safe_end = self._processed - self._max_secret_length + 1
+
+ emitted: list[str] = []
+ while self._pending_start < safe_end:
+ character = self._pending.popleft()
+ if self._position_is_covered(self._pending_start):
+ if not self._redaction_open:
+ emitted.append(self._replacement)
+ self._redaction_open = True
+ else:
+ self._redaction_open = False
+ emitted.append(character)
+ self._pending_start += 1
+ self._record_work()
+
+ return "".join(emitted)
+
+ def feed(self, text: str, *, final: bool = False) -> str:
+ """Return safe output, retaining at most one maximum-pattern window."""
+ if not self._has_secrets:
+ self._processed += len(text)
+ self._pending_start = self._processed
+ return text
+
+ emitted: list[str] = []
+ for character in text:
+ self._pending.append(character)
+ self._advance(character)
+ safe_output = self._drain(final=False)
+ if safe_output:
+ emitted.append(safe_output)
+ if final:
+ final_output = self._drain(final=True)
+ if final_output:
+ emitted.append(final_output)
+ return "".join(emitted)
+
+ def finish(self) -> str:
+ """Flush the final suffix once no later chunk can complete a match."""
+ return self.feed("", final=True)
async def _await_task_uninterruptibly(
@@ -83,6 +516,68 @@ def _validate_environment(environment: Mapping[str, str] | None) -> dict[str, st
return validated
+def _validate_compose_service_name(service: str) -> str:
+ if not isinstance(service, str) or not _COMPOSE_SERVICE_NAME_RE.fullmatch(service):
+ raise ValueError(f"Invalid Docker Compose service name: {service!r}")
+ return service
+
+
+def _compose_interpolation_names(content: str) -> set[str]:
+ """Extract Compose variable names while respecting ``$$`` escapes."""
+ names: set[str] = set()
+ index = 0
+ while index < len(content):
+ if content[index] != "$":
+ index += 1
+ continue
+ if index + 1 < len(content) and content[index + 1] == "$":
+ index += 2
+ continue
+ name_start = index + 1
+ if name_start < len(content) and content[name_start] == "{":
+ name_start += 1
+ match = _ENV_NAME_PREFIX_RE.match(content, name_start)
+ if match is None:
+ index += 1
+ continue
+ names.add(match.group())
+ index = match.end()
+ return names
+
+
+def _sidecar_environment_carriers(
+ environment: Mapping[str, str] | None,
+ *,
+ reserved_names: Iterable[str],
+) -> tuple[list[str], dict[str, str], str | None]:
+ """Map target env names through unpredictable client-safe carriers."""
+ validated = _validate_environment(environment)
+ if not validated:
+ return [], {}, None
+
+ occupied_names = set(reserved_names) | set(validated)
+ while True:
+ invocation_id = uuid.uuid4().hex.upper()
+ carrier_names = [f"{_SIDECAR_ENV_CARRIER_PREFIX}{invocation_id}_{index}" for index in range(len(validated))]
+ if occupied_names.isdisjoint(carrier_names):
+ break
+
+ carrier_environment = dict(zip(carrier_names, validated.values(), strict=True))
+ environment_args = [part for carrier in carrier_names for part in ("-e", carrier)]
+ exports = [
+ f'export {target_name}="${{{carrier_name}?missing sidecar environment carrier}}"'
+ for target_name, carrier_name in zip(validated, carrier_names, strict=True)
+ ]
+ wrapper = "; ".join(
+ (
+ *exports,
+ f"unset {' '.join(carrier_names)}",
+ 'exec /bin/sh -c "$1"',
+ )
+ )
+ return environment_args, carrier_environment, wrapper
+
+
def _secure_exec_arguments(
environment: Mapping[str, str] | None,
) -> tuple[list[str], dict[str, str]]:
@@ -92,207 +587,2440 @@ def _secure_exec_arguments(
return arguments, subprocess_environment
-def _redact(text: str | None, secret_values: set[str]) -> str | None:
- if text is None:
- return None
- redacted = text
- for value in sorted(
- (value for value in secret_values if value and len(value) >= _MIN_EXACT_SECRET_LENGTH),
- key=len,
- reverse=True,
- ):
- redacted = redacted.replace(value, "[REDACTED]")
- return redacted
+def _redact(
+ text: str | None,
+ secret_values: set[str],
+ *,
+ replacement: str | None = None,
+ include_short: bool = False,
+) -> str | None:
+ if text is None:
+ return None
+ redactor = _StreamingSecretRedactor(
+ secret_values,
+ _replacement=replacement,
+ _include_short=include_short,
+ )
+ return redactor.feed(text) + redactor.finish()
+
+
+def _redact_result(
+ result: ExecResult,
+ secret_values: set[str],
+ *,
+ replacement: str | None = None,
+ include_short: bool = False,
+) -> ExecResult:
+ return ExecResult(
+ stdout=_redact(
+ result.stdout,
+ secret_values,
+ replacement=replacement,
+ include_short=include_short,
+ ),
+ stderr=_redact(
+ result.stderr,
+ secret_values,
+ replacement=replacement,
+ include_short=include_short,
+ ),
+ return_code=result.return_code,
+ )
+
+
+def _signal_process_tree(process: asyncio.subprocess.Process, value: signal.Signals) -> None:
+ if process.returncode is not None:
+ return
+ if os.name == "posix":
+ try:
+ os.killpg(process.pid, value)
+ except ProcessLookupError:
+ return
+ except PermissionError:
+ # macOS can report EPERM instead of ESRCH if the process-group
+ # leader exits between the returncode check and killpg(). Suppress
+ # only when a second liveness check proves that PID is gone; a live
+ # process with a genuine permission failure must still fail closed.
+ if process.returncode is not None:
+ return
+ try:
+ os.getpgid(process.pid)
+ except ProcessLookupError:
+ return
+ raise
+ elif value == signal.SIGTERM:
+ process.terminate()
+ else:
+ process.kill()
+
+
+def _force_kill_process_tree(process: asyncio.subprocess.Process) -> None:
+ """Force-kill without evaluating POSIX-only signal constants on Windows."""
+ if os.name == "posix":
+ _signal_process_tree(process, signal.SIGKILL)
+ elif process.returncode is None:
+ process.kill()
+
+
+@contextlib.contextmanager
+def _raw_lifecycle_deadline_scope() -> Iterator[None]:
+ """Share one monotonic deadline across containment, reap, and restore."""
+ if _RAW_LIFECYCLE_DEADLINE.get() is not None:
+ yield
+ return
+ deadline = asyncio.get_running_loop().time() + _RAW_LIFECYCLE_TOTAL_TIMEOUT_SECONDS
+ token = _RAW_LIFECYCLE_DEADLINE.set(deadline)
+ try:
+ yield
+ finally:
+ _RAW_LIFECYCLE_DEADLINE.reset(token)
+
+
+def _bounded_cleanup_timeout(
+ maximum: float,
+ *,
+ allow_expired_reap: bool = False,
+) -> float:
+ deadline = _RAW_LIFECYCLE_DEADLINE.get()
+ if deadline is None:
+ return maximum
+ remaining = deadline - asyncio.get_running_loop().time()
+ if remaining <= 0:
+ if allow_expired_reap:
+ # A raw lifecycle deadline must never prevent the local Docker
+ # client from receiving SIGKILL and a final bounded reap attempt.
+ return max(0.001, min(maximum, _COMPOSE_CANCEL_SECONDS))
+ raise RuntimeError("Docker lifecycle cleanup deadline expired")
+ # Never return zero: several asyncio and Docker timeout APIs interpret it
+ # as an unlimited wait.
+ return max(0.001, min(maximum, remaining))
+
+
+async def _terminate_process_tree(
+ process: asyncio.subprocess.Process,
+ communication: asyncio.Task[Any],
+ *,
+ preserve_cancellation: bool,
+) -> None:
+ async def reap() -> None:
+ _signal_process_tree(process, signal.SIGTERM)
+ try:
+ terminate_timeout = _bounded_cleanup_timeout(_COMPOSE_TERMINATE_SECONDS)
+ except RuntimeError:
+ terminate_timeout = 0
+ done, _pending = await asyncio.wait({communication}, timeout=terminate_timeout)
+ if communication in done and process.returncode is not None:
+ with contextlib.suppress(asyncio.CancelledError, Exception):
+ communication.result()
+ return
+
+ _force_kill_process_tree(process)
+ done, _pending = await asyncio.wait(
+ {communication},
+ timeout=_bounded_cleanup_timeout(
+ _COMPOSE_KILL_SECONDS,
+ allow_expired_reap=True,
+ ),
+ )
+ if communication in done:
+ with contextlib.suppress(asyncio.CancelledError, Exception):
+ communication.result()
+ else:
+ communication.cancel()
+ done, _pending = await asyncio.wait(
+ {communication},
+ timeout=_bounded_cleanup_timeout(
+ _COMPOSE_CANCEL_SECONDS,
+ allow_expired_reap=True,
+ ),
+ )
+ if communication in done:
+ with contextlib.suppress(asyncio.CancelledError, Exception):
+ communication.result()
+ # asyncio cannot forcibly terminate a coroutine that deliberately
+ # suppresses CancelledError. Keep this wait bounded; cooperative
+ # callbacks are owned and reaped above, while a hostile callback can
+ # only be left for event-loop shutdown after ignoring cancellation.
+ if process.returncode is None:
+ raise RuntimeError("could not confirm Docker client process termination")
+
+ cleanup = asyncio.create_task(reap())
+ await _await_task_uninterruptibly(cleanup, preserve_cancellation=preserve_cancellation)
+
+
+def _host_handoff_environment(environment: Mapping[str, str]) -> dict[str, str]:
+ """Resolve a private NVIDIA Build sentinel without putting its value in argv."""
+ resolved = _validate_environment(environment)
+ if resolved.get("NVIDIA_API_KEY") == NVIDIA_BUILD_STDIN_SENTINEL:
+ resolved["NVIDIA_API_KEY"] = read_nvidia_build_key_from_stdin()
+ return resolved
+ if resolved.get("NVIDIA_API_KEY") != _NVIDIA_BUILD_FILE_SENTINEL:
+ return resolved
+ key_file = os.environ.get(_NVIDIA_BUILD_KEY_FILE_ENV, "").strip()
+ if not key_file:
+ raise RuntimeError(f"{_NVIDIA_BUILD_KEY_FILE_ENV} is required for NVIDIA Build Docker runs")
+ try:
+ api_key = Path(key_file).read_text(encoding="utf-8").strip()
+ except OSError as exc:
+ raise RuntimeError("NVIDIA Build key handoff file is unavailable") from exc
+ if not api_key:
+ raise RuntimeError("NVIDIA Build key handoff file is empty")
+ resolved["NVIDIA_API_KEY"] = api_key
+ return resolved
+
+
+def _render_environment_script(environment: Mapping[str, str]) -> str:
+ """Render a sourceable script after validating every name and value."""
+ validated = _validate_environment(environment)
+ lines = [f"export {name}={shlex.quote(value)}" for name, value in sorted(validated.items())]
+ return "\n".join(lines) + "\n"
+
+
+def _split_windows_container_file_source(source_path: str) -> tuple[str, str]:
+ """Split a container file path while preserving drive-root semantics."""
+ normalized = source_path.replace("\\", "/")
+ if not normalized or "\x00" in normalized or normalized.endswith("/"):
+ raise RuntimeError("requested Windows container download path is invalid")
+ parent, separator, name = normalized.rpartition("/")
+ if not name or name in {".", ".."}:
+ raise RuntimeError("requested Windows container download path is invalid")
+ if not separator:
+ parent = "."
+ elif not parent:
+ parent = "/"
+ elif re.fullmatch(r"[A-Za-z]:", parent):
+ parent += "/"
+ return parent, name
+
+
+@dataclass(frozen=True, slots=True)
+class _ValidatedTarMember:
+ canonical_parts: tuple[str, ...]
+ kind: str
+ size: int
+
+
+@dataclass(frozen=True, slots=True)
+class _WindowsArtifactUsage:
+ file_bytes: int
+ entries: int
+
+ @property
+ def estimated_disk_bytes(self) -> int:
+ return self.file_bytes + self.entries * _WINDOWS_ARTIFACT_ENTRY_DISK_BYTES
+
+
+@dataclass(frozen=True, slots=True)
+class _WindowsFilesystemReserve:
+ identity: int
+ minimum_free_bytes: int
+
+
+def _prescan_local_pax_payload(payload: bytes) -> int | None:
+ """Validate bounded PAX framing and return its effective size override."""
+ position = 0
+ record_count = 0
+ size_override: int | None = None
+ while position < len(payload):
+ if payload[position] == 0:
+ if any(payload[position:]):
+ raise ValueError
+ break
+
+ # Python's tarfile parser accepts one to twenty decimal digits followed
+ # by a space. Mirror that framing without building its raw-header list.
+ space = payload.find(b" ", position, min(len(payload), position + 21))
+ if space < 0:
+ raise ValueError
+ length_digits = payload[position:space]
+ if not length_digits or len(length_digits) > 20 or not length_digits.isdigit():
+ raise ValueError
+ record_length = int(length_digits)
+ record_end = position + record_length
+ if record_length < 5 or record_end > len(payload) or payload[record_end - 1] != 0x0A:
+ raise ValueError
+
+ equals = payload.find(b"=", space + 1, record_end - 1)
+ if equals <= space + 1:
+ raise ValueError
+ record_count += 1
+ if record_count > _WINDOWS_TAR_MAX_PAX_RECORDS_PER_HEADER:
+ raise ValueError
+
+ keyword = payload[space + 1 : equals]
+ if keyword.startswith(b"GNU.sparse."):
+ raise ValueError
+ if keyword == b"size":
+ raw_size = payload[equals + 1 : record_end - 1]
+ # POSIX pax size is an unsigned decimal. Bounding its spelling
+ # avoids turning a tiny metadata record into a large integer parse;
+ # twenty digits already covers every practical host file offset.
+ if not raw_size or len(raw_size) > 20 or not raw_size.isdigit():
+ raise ValueError
+ size_override = int(raw_size)
+ position = record_end
+ return size_override
+
+
+def _prescan_uncompressed_tar_archive(archive_path: Path) -> None:
+ """Bound tar metadata before ``tarfile`` is allowed to allocate it."""
+ extension_types = {
+ tarfile.XHDTYPE,
+ tarfile.XGLTYPE,
+ tarfile.SOLARIS_XHDTYPE,
+ tarfile.GNUTYPE_LONGNAME,
+ tarfile.GNUTYPE_LONGLINK,
+ }
+ archive_size = archive_path.stat().st_size
+ raw_members = 0
+ extension_bytes = 0
+ extension_chain = 0
+ pending_pax_size: int | None = None
+ saw_zero_block = False
+ with archive_path.open("rb") as archive:
+ while archive.tell() < archive_size:
+ header = archive.read(_TAR_BLOCK_BYTES)
+ if len(header) != _TAR_BLOCK_BYTES:
+ raise ValueError
+ if not any(header):
+ if extension_chain:
+ raise ValueError
+ if saw_zero_block:
+ while remainder := archive.read(64 * 1024):
+ if any(remainder):
+ raise ValueError
+ return
+ saw_zero_block = True
+ continue
+ if saw_zero_block:
+ raise ValueError
+ try:
+ member = tarfile.TarInfo.frombuf(
+ header,
+ encoding="utf-8",
+ errors="surrogateescape",
+ )
+ except (tarfile.HeaderError, ValueError):
+ raise ValueError from None
+ raw_members += 1
+ if raw_members > _WINDOWS_TAR_MAX_MEMBERS:
+ raise ValueError
+ if member.size < 0 or member.type == tarfile.GNUTYPE_SPARSE:
+ raise ValueError
+ # Global PAX state is copied into every later TarInfo by the
+ # stdlib parser, creating multiplicative CPU and memory cost.
+ if member.type == tarfile.XGLTYPE:
+ raise ValueError
+ is_extension = member.type in extension_types
+ if is_extension:
+ extension_chain += 1
+ if extension_chain > _WINDOWS_TAR_MAX_EXTENSION_CHAIN:
+ raise ValueError
+ if member.size > _WINDOWS_TAR_MAX_EXTENSION_BYTES:
+ raise ValueError
+ extension_bytes += member.size
+ if extension_bytes > _WINDOWS_TAR_MAX_TOTAL_EXTENSION_BYTES:
+ raise ValueError
+ else:
+ extension_chain = 0
+ effective_size = member.size
+ if not is_extension and pending_pax_size is not None:
+ # ``TarInfo._proc_pax`` recalculates the next raw-header offset
+ # only for regular or unsupported member types. With stacked
+ # extension headers, the outermost (first) local pax value is
+ # applied last and therefore wins.
+ if member.isreg() or member.type not in tarfile.SUPPORTED_TYPES:
+ effective_size = pending_pax_size
+ pending_pax_size = None
+ padded_size = ((effective_size + _TAR_BLOCK_BYTES - 1) // _TAR_BLOCK_BYTES) * _TAR_BLOCK_BYTES
+ if archive.tell() + padded_size > archive_size:
+ raise ValueError
+ if member.type in {tarfile.XHDTYPE, tarfile.SOLARIS_XHDTYPE}:
+ payload = archive.read(member.size)
+ if len(payload) != member.size:
+ raise ValueError
+ size_override = _prescan_local_pax_payload(payload)
+ if pending_pax_size is None:
+ pending_pax_size = size_override
+ archive.seek(padded_size - member.size, os.SEEK_CUR)
+ else:
+ archive.seek(padded_size, os.SEEK_CUR)
+ raise ValueError
+
+
+def _validated_windows_tar_member(
+ member: tarfile.TarInfo,
+ target: Path,
+) -> _ValidatedTarMember | None:
+ """Return one collision-safe Windows pathname or reject the member."""
+ name = member.name
+ if not name or "\x00" in name or "\\" in name:
+ raise ValueError
+ try:
+ raw_name_bytes = len(name.encode("utf-8", errors="strict"))
+ except UnicodeEncodeError:
+ raise ValueError from None
+ if raw_name_bytes > _WINDOWS_TAR_MAX_MEMBER_PATH_BYTES:
+ raise ValueError
+ declared = PurePosixPath(name)
+ if declared.is_absolute():
+ raise ValueError
+
+ spelling = name[:-1] if member.isdir() and name.endswith("/") else name
+ raw_parts = spelling.split("/")
+ if len(raw_parts) > _WINDOWS_TAR_MAX_DEPTH + _WINDOWS_TAR_MAX_LEADING_CURRENT_COMPONENTS:
+ raise ValueError
+ first_component = 0
+ while first_component < len(raw_parts) and raw_parts[first_component] == ".":
+ first_component += 1
+ if first_component > _WINDOWS_TAR_MAX_LEADING_CURRENT_COMPONENTS:
+ raise ValueError
+ raw_parts = raw_parts[first_component:]
+ if any(part in {"", ".", ".."} for part in raw_parts):
+ raise ValueError
+ if not raw_parts:
+ if not member.isdir():
+ raise ValueError
+ return None
+ if len(raw_parts) > _WINDOWS_TAR_MAX_DEPTH:
+ raise ValueError
+
+ canonical_parts: list[str] = []
+ for part in raw_parts:
+ if part.endswith((" ", ".")) or ":" in part:
+ raise ValueError
+ normalized = unicodedata.normalize("NFC", part)
+ try:
+ normalized.encode("utf-8", errors="strict")
+ except UnicodeEncodeError:
+ raise ValueError from None
+ reserved_stem = normalized.split(".", maxsplit=1)[0]
+ if _WINDOWS_RESERVED_NAME_RE.fullmatch(reserved_stem):
+ raise ValueError
+ canonical_parts.append(normalized.casefold())
+
+ filtered = tarfile.data_filter(member, str(target))
+ if filtered is None or getattr(filtered, "sparse", None):
+ raise ValueError
+ if filtered.isdir():
+ kind = "directory"
+ elif filtered.isfile() and filtered.size >= 0:
+ kind = "file"
+ else:
+ raise ValueError
+ return _ValidatedTarMember(tuple(canonical_parts), kind, filtered.size)
+
+
+def _windows_existing_ancestor(path: Path) -> Path:
+ """Return the canonical nearest existing ancestor of a host path."""
+ probe = _absolute_lexical(path)
+ while True:
+ try:
+ probe.lstat()
+ return probe
+ except FileNotFoundError:
+ parent = probe.parent
+ if parent == probe:
+ raise
+ probe = parent
+
+
+def _windows_artifact_disk_budget(path: Path) -> tuple[int, int]:
+ """Return the current free bytes and an adaptive initial byte budget."""
+ free_bytes = shutil.disk_usage(_windows_existing_ancestor(path)).free
+ # A reserve must not ratchet downward as spool, extraction, and publication
+ # consume one filesystem. Callers snapshot this adaptive initial value and
+ # reuse it for the operation.
+ reserve_bytes = min(
+ _WINDOWS_ARTIFACT_DISK_RESERVE_BYTES,
+ max(64 * 1024 * 1024, free_bytes // 20),
+ )
+ return free_bytes, max(0, free_bytes - reserve_bytes)
+
+
+def _windows_artifact_filesystem_reserve(path: Path) -> _WindowsFilesystemReserve:
+ """Snapshot one filesystem's minimum free headroom for an operation."""
+ probe = _windows_existing_ancestor(path)
+ free_bytes, budget_bytes = _windows_artifact_disk_budget(probe)
+ return _WindowsFilesystemReserve(
+ identity=probe.stat().st_dev,
+ minimum_free_bytes=free_bytes - budget_bytes,
+ )
+
+
+def _windows_artifact_inode_budget(path: Path) -> int | None:
+ """Return a headroom-preserving inode budget when the host exposes one."""
+ statvfs = getattr(os, "statvfs", None)
+ if statvfs is None:
+ return None
+ filesystem = statvfs(_windows_existing_ancestor(path))
+ if filesystem.f_files <= 0 or filesystem.f_favail < 0:
+ return None
+ reserve = min(_WINDOWS_ARTIFACT_INODE_RESERVE, filesystem.f_favail)
+ return max(0, filesystem.f_favail - reserve)
+
+
+def _require_windows_artifact_resources(
+ path: Path,
+ required_bytes: int,
+ *,
+ required_entries: int,
+ minimum_free_bytes: int | None = None,
+ purpose: str,
+) -> None:
+ """Fail before mutation when a filesystem cannot retain headroom."""
+ _free_bytes, budget_bytes = _windows_artifact_disk_budget(path)
+ if minimum_free_bytes is not None:
+ budget_bytes = max(0, _free_bytes - minimum_free_bytes)
+ if required_bytes > budget_bytes:
+ raise RuntimeError(f"insufficient disk space for secure Windows container {purpose}")
+ inode_budget = _windows_artifact_inode_budget(path)
+ if inode_budget is not None and required_entries > inode_budget:
+ raise RuntimeError(f"insufficient filesystem entries for secure Windows container {purpose}")
+
+
+def _existing_windows_target_usage(path: Path, *, kind: str) -> _WindowsArtifactUsage:
+ """Conservatively estimate transactional copies of an existing target."""
+ try:
+ root = path.lstat()
+ except FileNotFoundError:
+ return _WindowsArtifactUsage(file_bytes=0, entries=0)
+ except OSError:
+ raise RuntimeError("requested Windows container download target is unsafe") from None
+
+ try:
+ if stat_is_link_or_reparse(root):
+ raise ValueError
+ if kind == "file":
+ if not stat.S_ISREG(root.st_mode) or root.st_nlink != 1:
+ raise ValueError
+ return _WindowsArtifactUsage(file_bytes=root.st_size, entries=1)
+ if kind != "directory" or not stat.S_ISDIR(root.st_mode):
+ raise ValueError
+
+ root_device = root.st_dev
+ entry_count = 1
+ file_bytes = 0
+ pending = [path]
+ while pending:
+ directory = pending.pop()
+ with os.scandir(directory) as children:
+ for child in children:
+ metadata = child.stat(follow_symlinks=False)
+ entry_count += 1
+ if entry_count > _WINDOWS_TAR_MAX_FILESYSTEM_ENTRIES:
+ raise ValueError
+ if stat_is_link_or_reparse(metadata) or metadata.st_dev != root_device:
+ raise ValueError
+ child_path = Path(child.path)
+ if stat.S_ISDIR(metadata.st_mode):
+ pending.append(child_path)
+ elif stat.S_ISREG(metadata.st_mode) and metadata.st_nlink == 1:
+ file_bytes += metadata.st_size
+ else:
+ raise ValueError
+ return _WindowsArtifactUsage(file_bytes=file_bytes, entries=entry_count)
+ except (OSError, ValueError):
+ raise RuntimeError("requested Windows container download target is unsafe") from None
+
+
+def _missing_windows_target_parent_usage(path: Path) -> _WindowsArtifactUsage:
+ """Validate destination ancestors and charge parents that must be created."""
+ missing_entries = 0
+ probe = _absolute_lexical(path).parent
+ while True:
+ try:
+ metadata = probe.lstat()
+ except FileNotFoundError:
+ missing_entries += 1
+ parent = probe.parent
+ if parent == probe:
+ raise RuntimeError("requested Windows container download target is unsafe") from None
+ probe = parent
+ continue
+ except OSError:
+ raise RuntimeError("requested Windows container download target is unsafe") from None
+ if stat_is_link_or_reparse(metadata) or not stat.S_ISDIR(metadata.st_mode):
+ raise RuntimeError("requested Windows container download target is unsafe")
+ parent = probe.parent
+ if parent == probe:
+ return _WindowsArtifactUsage(file_bytes=0, entries=missing_entries)
+ probe = parent
+
+
+def _extract_regular_tar_archive(
+ archive_path: Path,
+ target_dir: Path | str,
+ *,
+ minimum_free_bytes: int | None = None,
+) -> _WindowsArtifactUsage:
+ """Validate a disk-backed container archive, then extract it privately."""
+ target = Path(target_dir)
+ try:
+ expected_target = target.lstat()
+ if stat_is_link_or_reparse(expected_target) or not stat.S_ISDIR(expected_target.st_mode):
+ raise ValueError
+ archive_metadata = archive_path.lstat()
+ if (
+ stat_is_link_or_reparse(archive_metadata)
+ or not stat.S_ISREG(archive_metadata.st_mode)
+ or archive_metadata.st_nlink != 1
+ ):
+ raise ValueError
+ _prescan_uncompressed_tar_archive(archive_path)
+
+ manifest: list[_ValidatedTarMember] = []
+ member_kinds: dict[tuple[str, ...], str] = {}
+ path_bytes = 0
+ path_components = 0
+ total_file_bytes = 0
+ with tarfile.open(archive_path, mode="r:", errorlevel=2) as archive:
+ for member in archive:
+ validated = _validated_windows_tar_member(member, target)
+ archive.members.clear()
+ if validated is None:
+ continue
+ path_bytes += sum(len(part.encode("utf-8")) for part in validated.canonical_parts)
+ path_components += len(validated.canonical_parts)
+ if (
+ len(manifest) >= _WINDOWS_TAR_MAX_MEMBERS
+ or path_bytes > _WINDOWS_TAR_MAX_PATH_BYTES
+ or path_components > _WINDOWS_TAR_MAX_PATH_COMPONENTS
+ ):
+ raise ValueError
+ key = validated.canonical_parts
+ if key in member_kinds:
+ raise ValueError
+ if validated.kind == "file":
+ total_file_bytes += validated.size
+ member_kinds[key] = validated.kind
+ manifest.append(validated)
+
+ ordered_keys = sorted(member_kinds)
+ for index, key in enumerate(ordered_keys[:-1]):
+ following = ordered_keys[index + 1]
+ if member_kinds[key] == "file" and len(following) > len(key) and following[: len(key)] == key:
+ raise ValueError
+
+ # Count implicit parent directories without retaining every prefix.
+ # Adjacent sorted paths share all prefixes that could already exist.
+ filesystem_entries = 1 # private extraction/publication root
+ previous_key: tuple[str, ...] = ()
+ for key in ordered_keys:
+ common_depth = 0
+ for left, right in zip(previous_key, key, strict=False):
+ if left != right:
+ break
+ common_depth += 1
+ directory_depth = len(key) if member_kinds[key] == "directory" else len(key) - 1
+ filesystem_entries += max(0, directory_depth - common_depth)
+ if member_kinds[key] == "file":
+ filesystem_entries += 1
+ previous_key = key
+
+ if filesystem_entries > _WINDOWS_TAR_MAX_FILESYSTEM_ENTRIES:
+ raise ValueError
+ usage = _WindowsArtifactUsage(
+ file_bytes=total_file_bytes,
+ entries=filesystem_entries,
+ )
+ _require_windows_artifact_resources(
+ target,
+ usage.estimated_disk_bytes,
+ required_entries=usage.entries,
+ minimum_free_bytes=minimum_free_bytes,
+ purpose="archive extraction",
+ )
+
+ extracted_index = 0
+ with tarfile.open(archive_path, mode="r|", errorlevel=2) as archive:
+ for member in archive:
+ validated = _validated_windows_tar_member(member, target)
+ archive.members.clear()
+ if validated is None:
+ continue
+ if extracted_index >= len(manifest) or validated != manifest[extracted_index]:
+ raise ValueError
+ archive.extract(member, path=target, filter="data")
+ extracted_index += 1
+ if extracted_index != len(manifest):
+ raise ValueError
+
+ observed_target = target.lstat()
+ observed_archive = archive_path.lstat()
+ if (
+ stat_is_link_or_reparse(observed_target)
+ or not stat.S_ISDIR(observed_target.st_mode)
+ or not os.path.samestat(expected_target, observed_target)
+ or stat_is_link_or_reparse(observed_archive)
+ or not stat.S_ISREG(observed_archive.st_mode)
+ or not os.path.samestat(archive_metadata, observed_archive)
+ ):
+ raise ValueError
+ return usage
+ except (OSError, RecursionError, tarfile.TarError, UnicodeError, ValueError):
+ raise RuntimeError("unsafe Windows container download archive") from None
+
+
+class SkillEvaluatorDockerEnvironment(DockerEnvironment):
+ """Pinned Harbor compatibility backend with host-visible argv safety."""
+
+ @classmethod
+ def preflight(cls) -> None:
+ """Consume the private stdin handoff before Docker can inherit it."""
+ if os.environ.get("NVIDIA_API_KEY", "").strip() == NVIDIA_BUILD_STDIN_SENTINEL:
+ read_nvidia_build_key_from_stdin()
+ super().preflight()
+
+ async def start(self, force_build: bool) -> None:
+ """Reject unsupported Compose inputs before Docker mutates a project."""
+ self._compose_model_metadata()
+ await super().start(force_build)
+
+ @staticmethod
+ async def _collect_streamed_output(
+ process: asyncio.subprocess.Process,
+ *,
+ timeout_sec: int | None,
+ stdin_data: bytes | None = None,
+ on_output: OutputCallback,
+ ) -> ExecResult:
+ """Stream output within one hard raw-byte budget."""
+ stdout_stream = process.stdout
+ if stdout_stream is None:
+ raise RuntimeError("Streaming requires a captured stdout pipe")
+ output = bytearray()
+ decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+ output_budget = CommandOutputByteBudget()
+
+ async def write_stdin() -> None:
+ if stdin_data is None:
+ return
+ stdin = process.stdin
+ if stdin is None:
+ raise RuntimeError("stdin_data requires a stdin pipe")
+ stdin.write(stdin_data)
+ await stdin.drain()
+ stdin.close()
+ await stdin.wait_closed()
+
+ async def read_stdout_and_wait() -> None:
+ while raw_chunk := await stdout_stream.read(64 * 1024):
+ output_budget.consume(raw_chunk)
+ output.extend(raw_chunk)
+ if text := decoder.decode(raw_chunk):
+ await on_output(text, "stdout")
+ await asyncio.sleep(0)
+ if text := decoder.decode(b"", final=True):
+ await on_output(text, "stdout")
+ await process.wait()
+
+ async def read_and_wait() -> None:
+ if stdin_data is not None:
+ tasks = [
+ asyncio.create_task(write_stdin()),
+ asyncio.create_task(read_stdout_and_wait()),
+ ]
+ try:
+ await asyncio.gather(*tasks)
+ except BaseException:
+ for task in tasks:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ raise
+ else:
+ await read_stdout_and_wait()
+
+ try:
+ if timeout_sec:
+ await asyncio.wait_for(read_and_wait(), timeout=timeout_sec)
+ else:
+ await read_and_wait()
+ except TimeoutError:
+ await DockerEnvironment._terminate_process(process)
+ raise RuntimeError(f"Command timed out after {timeout_sec} seconds") from None
+ except BaseException:
+ if process.returncode is None:
+ await DockerEnvironment._terminate_process(process)
+ raise
+
+ stdout = output.decode(errors="replace")
+ return ExecResult(
+ stdout=stdout or None,
+ stderr=None,
+ return_code=process.returncode or 0,
+ )
+
+ def _trusted_docker_client_environment(self) -> dict[str, str]:
+ """Build a host-only Docker CLI environment without Compose/task state."""
+ trusted_environment: dict[str, str] = {}
+ for name, value in os.environ.items():
+ if name.upper().startswith("COMPOSE_") or not self._is_trusted_compose_client_host_name(name):
+ continue
+ # A task override with the same name must not remove the genuine
+ # host Docker control. The value here comes only from os.environ;
+ # coincidental byte overlap with attacker-chosen task values cannot
+ # be allowed to disable host-authoritative containment.
+ trusted_environment[name] = value
+ if "PATH" not in trusted_environment:
+ raise RuntimeError("trusted Docker client PATH is unavailable")
+ return trusted_environment
+
+ async def _run_trusted_docker_command(
+ self,
+ command: list[str],
+ ) -> ExecResult:
+ """Run a bounded raw-Docker command under the scrubbed host baseline."""
+ process_environment = self._trusted_docker_client_environment()
+ docker_executable = shutil.which(
+ "docker",
+ path=process_environment.get("PATH"),
+ )
+ if docker_executable is None:
+ raise RuntimeError("trusted Docker client executable was not found")
+ full_command = [docker_executable, *command]
+ creation_timeout = _bounded_cleanup_timeout(_RAW_DOCKER_COMMAND_TIMEOUT_SECONDS)
+ creation = asyncio.create_task(
+ asyncio.create_subprocess_exec(
+ *full_command,
+ env=process_environment,
+ stdin=asyncio.subprocess.DEVNULL,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ start_new_session=os.name == "posix",
+ )
+ )
+ try:
+ process = await asyncio.wait_for(
+ asyncio.shield(creation),
+ timeout=creation_timeout,
+ )
+ except (TimeoutError, asyncio.CancelledError) as primary_error:
+
+ async def reap_late_creation(
+ completed_creation: asyncio.Task[asyncio.subprocess.Process],
+ ) -> None:
+ try:
+ late_process = completed_creation.result()
+ except BaseException:
+ return
+ late_communication = asyncio.create_task(late_process.communicate())
+ await _terminate_process_tree(
+ late_process,
+ late_communication,
+ preserve_cancellation=False,
+ )
+
+ def schedule_late_reap(
+ completed_creation: asyncio.Task[asyncio.subprocess.Process],
+ ) -> None:
+ late_cleanup = asyncio.create_task(reap_late_creation(completed_creation))
+
+ def report_late_cleanup_failure(
+ completed_cleanup: asyncio.Task[None],
+ ) -> None:
+ try:
+ completed_cleanup.result()
+ except BaseException as exc:
+ asyncio.get_running_loop().call_exception_handler(
+ {
+ "message": "late Docker client creation cleanup failed",
+ "exception": exc,
+ "task": completed_cleanup,
+ }
+ )
+
+ late_cleanup.add_done_callback(report_late_cleanup_failure)
+
+ async def cancel_creation_race() -> tuple[
+ asyncio.subprocess.Process | None,
+ bool,
+ ]:
+ creation.cancel()
+ try:
+ cancellation_timeout = _bounded_cleanup_timeout(_COMPOSE_CANCEL_SECONDS)
+ except RuntimeError:
+ cancellation_timeout = 0
+ done, _pending = await asyncio.wait(
+ {creation},
+ timeout=cancellation_timeout,
+ )
+ if creation not in done:
+ creation.add_done_callback(schedule_late_reap)
+ return None, False
+ try:
+ return creation.result(), True
+ except BaseException:
+ return None, True
+
+ cancellation = asyncio.create_task(cancel_creation_race())
+ process, creation_resolved = await _await_task_uninterruptibly(
+ cancellation,
+ preserve_cancellation=False,
+ )
+ if process is not None:
+ communication = asyncio.create_task(process.communicate())
+ await _terminate_process_tree(
+ process,
+ communication,
+ preserve_cancellation=False,
+ )
+ if not creation_resolved:
+ primary_error.add_note(
+ "Docker client creation cancellation remained pending past the cleanup deadline; "
+ "a late-process reaper was installed"
+ )
+ if isinstance(primary_error, TimeoutError):
+ raise RuntimeError("trusted Docker client creation timed out") from primary_error
+ raise
+
+ communication = asyncio.create_task(process.communicate())
+ try:
+ communication_timeout = _bounded_cleanup_timeout(_RAW_DOCKER_COMMAND_TIMEOUT_SECONDS)
+ done, _pending = await asyncio.wait(
+ {communication},
+ timeout=communication_timeout,
+ )
+ if communication not in done:
+ raise _ComposeCommandTimeout
+ stdout_bytes, stderr_bytes = communication.result()
+ except BaseException as primary_error:
+ await _terminate_process_tree(
+ process,
+ communication,
+ preserve_cancellation=False,
+ )
+ if isinstance(primary_error, _ComposeCommandTimeout):
+ raise RuntimeError("trusted Docker client command timed out") from primary_error
+ raise
+ return ExecResult(
+ stdout=(stdout_bytes.decode(errors="replace") if stdout_bytes else None),
+ stderr=(stderr_bytes.decode(errors="replace") if stderr_bytes else None),
+ return_code=process.returncode or 0,
+ )
+
+ async def _raw_filtered_service_container_ids(self, service: str) -> tuple[str, ...]:
+ """Resolve exact IDs from Docker's authoritative Compose-label index."""
+ service = _validate_compose_service_name(service)
+ project = _sanitize_docker_compose_project_name(self.session_id)
+ result = await self._run_trusted_docker_command(
+ [
+ "container",
+ "ls",
+ "--all",
+ "--quiet",
+ "--no-trunc",
+ "--filter",
+ f"label=com.docker.compose.project={project}",
+ "--filter",
+ f"label=com.docker.compose.service={service}",
+ "--filter",
+ "label=com.docker.compose.oneoff=False",
+ "--filter",
+ "label=com.docker.compose.config-hash",
+ ]
+ )
+ if result.return_code != 0:
+ raise RuntimeError("could not resolve Docker service containers")
+ identities = tuple(line.strip() for line in (result.stdout or "").splitlines() if line.strip())
+ if len(set(identities)) != len(identities) or any(
+ not re.fullmatch(r"[0-9a-f]{64}", identity, re.IGNORECASE) for identity in identities
+ ):
+ raise RuntimeError("Docker returned an invalid service container identity")
+ return identities
+
+ async def _raw_service_container_ids(self, service: str) -> tuple[str, ...]:
+ """Resolve and validate every managed container for one service."""
+ service = _validate_compose_service_name(service)
+ identities = await self._raw_filtered_service_container_ids(service)
+ states = await self._raw_container_states(identities, service=service)
+ if len({state.container_number for state in states.values()}) != len(states):
+ raise RuntimeError("Docker returned duplicate service container numbers")
+ return tuple(
+ state.identity
+ for state in sorted(
+ states.values(),
+ key=lambda state: state.container_number,
+ )
+ )
+
+ async def _raw_container_states(
+ self,
+ identities: Iterable[str],
+ *,
+ service: str,
+ ) -> dict[str, _RawContainerState]:
+ service = _validate_compose_service_name(service)
+ validated_identities = tuple(identities)
+ if not validated_identities:
+ return {}
+ if any(not re.fullmatch(r"[0-9a-f]{64}", identity, re.IGNORECASE) for identity in validated_identities):
+ raise RuntimeError("invalid Docker container identity")
+ result = await self._run_trusted_docker_command(
+ [
+ "container",
+ "inspect",
+ "--format",
+ (
+ '{{.Id}}\t{{index .Config.Labels "com.docker.compose.project"}}'
+ '\t{{index .Config.Labels "com.docker.compose.service"}}'
+ '\t{{index .Config.Labels "com.docker.compose.container-number"}}'
+ '\t{{index .Config.Labels "com.docker.compose.oneoff"}}'
+ '\t{{index .Config.Labels "com.docker.compose.config-hash"}}'
+ "\t{{.State.Running}}\t{{.State.Paused}}"
+ "\t{{.State.Restarting}}\t{{.State.Status}}"
+ '\t{{with index .State "Health"}}{{.Status}}{{else}}none{{end}}'
+ ),
+ "--",
+ *validated_identities,
+ ]
+ )
+ if result.return_code != 0:
+ raise RuntimeError("could not inspect Docker service containers")
+ lines = (result.stdout or "").splitlines()
+ if len(lines) != len(validated_identities):
+ raise RuntimeError("Docker returned incomplete container state")
+ project = _sanitize_docker_compose_project_name(self.session_id)
+ states: dict[str, _RawContainerState] = {}
+ for identity, line in zip(validated_identities, lines, strict=True):
+ fields = line.split("\t")
+ if len(fields) != 11:
+ raise RuntimeError("Docker returned invalid container state")
+ (
+ rendered_identity,
+ rendered_project,
+ rendered_service,
+ rendered_number,
+ rendered_oneoff,
+ rendered_config_hash,
+ rendered_running,
+ rendered_paused,
+ rendered_restarting,
+ rendered_status,
+ rendered_health,
+ ) = fields
+ if (
+ rendered_identity.lower() != identity.lower()
+ or rendered_project != project
+ or rendered_service != service
+ or rendered_oneoff != "False"
+ or re.fullmatch(r"[0-9a-f]{64}", rendered_config_hash, re.IGNORECASE) is None
+ or re.fullmatch(r"[1-9][0-9]*", rendered_number) is None
+ or rendered_running not in {"true", "false"}
+ or rendered_paused not in {"true", "false"}
+ or rendered_restarting not in {"true", "false"}
+ or rendered_status
+ not in {
+ "created",
+ "running",
+ "paused",
+ "restarting",
+ "removing",
+ "exited",
+ "dead",
+ }
+ or rendered_health not in {"none", "starting", "healthy", "unhealthy"}
+ ):
+ raise RuntimeError("Docker returned invalid container state")
+ states[identity] = _RawContainerState(
+ identity=identity,
+ project=rendered_project,
+ service=rendered_service,
+ container_number=int(rendered_number),
+ running=rendered_running == "true",
+ paused=rendered_paused == "true",
+ restarting=rendered_restarting == "true",
+ status=rendered_status,
+ health_status=(None if rendered_health == "none" else rendered_health),
+ )
+ return states
+
+ async def _raw_docker_action(
+ self,
+ action: list[str],
+ identities: Iterable[str],
+ ) -> bool:
+ validated_identities = tuple(identities)
+ if not validated_identities:
+ return True
+ result = await self._run_trusted_docker_command([*action, "--", *validated_identities])
+ return result.return_code == 0
+
+ async def _stop_raw_service_containers(
+ self,
+ service: str,
+ *,
+ remove: bool,
+ require_existing: bool,
+ ) -> _RawServiceSnapshot:
+ service = _validate_compose_service_name(service)
+ stop_failure: BaseException | None = None
+ try:
+ initial_identities = await self._raw_service_container_ids(service)
+ initial_states = await self._raw_container_states(
+ initial_identities,
+ service=service,
+ )
+ except BaseException as exc:
+ if not remove:
+ raise
+ stop_failure = exc
+ initial_identities = await self._raw_filtered_service_container_ids(service)
+ initial_states = {}
+ if require_existing and not initial_identities:
+ raise RuntimeError(f"could not resolve a container for sidecar service {service!r}")
+ if not initial_identities:
+ return _RawServiceSnapshot((), ())
+ restore_identities = tuple(
+ identity
+ for identity in initial_identities
+ if identity in initial_states and initial_states[identity].running
+ )
+
+ try:
+ if stop_failure is not None:
+ raise stop_failure
+ for action in (
+ ["container", "stop", "--timeout", "0"],
+ ["container", "kill", "--signal", "SIGKILL"],
+ ):
+ current_identities = await self._raw_service_container_ids(service)
+ states = await self._raw_container_states(
+ current_identities,
+ service=service,
+ )
+ running_identities = tuple(identity for identity, state in states.items() if state.running)
+ if not running_identities:
+ break
+ await self._raw_docker_action(action, running_identities)
+
+ current_identities = await self._raw_service_container_ids(service)
+ states = await self._raw_container_states(
+ current_identities,
+ service=service,
+ )
+ if any(state.running or state.restarting or state.paused for state in states.values()):
+ raise RuntimeError(f"could not confirm Docker service {service!r} stopped")
+ except BaseException as exc:
+ if not remove:
+ raise
+ stop_failure = exc
+
+ if remove:
+ removal_failure: BaseException | None = stop_failure
+ removal_candidates = set(initial_identities)
+ for _attempt in range(2):
+ try:
+ removal_candidates = set(await self._raw_filtered_service_container_ids(service))
+ except BaseException as exc:
+ removal_failure = exc
+ if not removal_candidates:
+ break
+ try:
+ await self._raw_docker_action(
+ ["container", "rm", "--force", "--volumes"],
+ removal_candidates,
+ )
+ except BaseException as exc:
+ removal_failure = exc
+ try:
+ remaining_identities = await self._raw_filtered_service_container_ids(service)
+ except BaseException as exc:
+ raise RuntimeError(f"could not confirm Docker service {service!r} removal") from exc
+ if remaining_identities:
+ error = RuntimeError(f"could not confirm Docker service {service!r} removal")
+ if removal_failure is not None:
+ raise error from removal_failure
+ raise error
+ return _RawServiceSnapshot(initial_identities, restore_identities)
+
+ async def _contain_main_container(self) -> None:
+ """Remove only this project's main container after an interrupted exec."""
+ await self._stop_raw_service_containers(
+ MAIN_SERVICE_NAME,
+ remove=True,
+ require_existing=False,
+ )
+
+ async def _restore_sidecar_service(
+ self,
+ service: str,
+ *,
+ snapshot: _RawServiceSnapshot,
+ ) -> bool:
+ service = _validate_compose_service_name(service)
+ identities = snapshot.running_identities
+ if not identities:
+ return False
+ current_identities = set(await self._raw_service_container_ids(service))
+ if current_identities != set(snapshot.all_identities):
+ return False
+ states = await self._raw_container_states(
+ snapshot.all_identities,
+ service=service,
+ )
+ if any(state.running or state.restarting for state in states.values()):
+ return False
+ if not await self._raw_docker_action(["container", "start"], identities):
+ return False
+
+ while True:
+ if set(await self._raw_service_container_ids(service)) != set(snapshot.all_identities):
+ return False
+ states = await self._raw_container_states(
+ snapshot.all_identities,
+ service=service,
+ )
+ ready = True
+ for identity in snapshot.all_identities:
+ state = states[identity]
+ if identity not in snapshot.running_identities:
+ if state.running or state.paused or state.restarting:
+ return False
+ continue
+ if state.health_status == "unhealthy" or state.status in {"created", "removing", "exited", "dead"}:
+ return False
+ if (
+ not state.running
+ or state.paused
+ or state.restarting
+ or state.status != "running"
+ or state.health_status not in {None, "healthy"}
+ ):
+ ready = False
+ if ready and len(states) == len(snapshot.all_identities):
+ return True
+ await asyncio.sleep(_bounded_cleanup_timeout(0.1))
+
+ async def _contain_sidecar_service(self, service: str) -> _RawServiceSnapshot:
+ """Stop only the target sidecar and retain its exact IDs for restart."""
+ service = _validate_compose_service_name(service)
+ if service == MAIN_SERVICE_NAME:
+ raise ValueError("sidecar containment cannot target the main service")
+ return await self._stop_raw_service_containers(
+ service,
+ remove=False,
+ require_existing=True,
+ )
+
+ async def _contain_main_and_reap_compose(
+ self,
+ process: asyncio.subprocess.Process,
+ communication: asyncio.Task[Any],
+ *,
+ contain_service_on_interrupt: str | None = None,
+ stop_main_on_interrupt: bool,
+ ) -> None:
+ with _raw_lifecycle_deadline_scope():
+ await self._contain_main_and_reap_compose_within_deadline(
+ process,
+ communication,
+ contain_service_on_interrupt=contain_service_on_interrupt,
+ stop_main_on_interrupt=stop_main_on_interrupt,
+ )
+
+ async def _contain_main_and_reap_compose_within_deadline(
+ self,
+ process: asyncio.subprocess.Process,
+ communication: asyncio.Task[Any],
+ *,
+ contain_service_on_interrupt: str | None,
+ stop_main_on_interrupt: bool,
+ ) -> None:
+ containment_error: BaseException | None = None
+ reap_error: BaseException | None = None
+ restoration_error: BaseException | None = None
+ sidecar_snapshot: _RawServiceSnapshot | None = None
+ if stop_main_on_interrupt and contain_service_on_interrupt is not None:
+ raise ValueError("only one interrupt-containment target may be configured")
+ if stop_main_on_interrupt:
+ try:
+ await self._contain_main_container()
+ except BaseException as exc:
+ containment_error = exc
+ elif contain_service_on_interrupt is not None:
+ try:
+ sidecar_snapshot = await self._contain_sidecar_service(contain_service_on_interrupt)
+ except BaseException as exc:
+ containment_error = exc
+ try:
+ await _terminate_process_tree(
+ process,
+ communication,
+ preserve_cancellation=False,
+ )
+ except BaseException as exc:
+ reap_error = exc
+
+ if (
+ containment_error is None
+ and reap_error is None
+ and contain_service_on_interrupt is not None
+ and sidecar_snapshot is not None
+ ):
+ try:
+ restored = await self._restore_sidecar_service(
+ contain_service_on_interrupt,
+ snapshot=sidecar_snapshot,
+ )
+ if not restored:
+ raise RuntimeError(f"sidecar service {contain_service_on_interrupt!r} could not be restored")
+ except BaseException as exc:
+ restoration_error = exc
+
+ failures = [error for error in (containment_error, reap_error, restoration_error) if error is not None]
+ if failures:
+ target = (
+ "main task container" if stop_main_on_interrupt else f"sidecar service {contain_service_on_interrupt!r}"
+ )
+ error = RuntimeError(f"could not confirm {target} containment and restoration")
+ for additional_error in failures[1:]:
+ error.add_note(
+ f"Additional containment cleanup failure: {type(additional_error).__name__}: {additional_error}"
+ )
+ raise error from failures[0]
+
+ async def exec(
+ self,
+ command: str,
+ cwd: str | None = None,
+ env: dict[str, str] | None = None,
+ timeout_sec: int | None = None,
+ user: str | int | None = None,
+ ) -> ExecResult:
+ user = self._resolve_user(user)
+ merged_environment = self._merge_env(env)
+ environment_args, subprocess_environment = _secure_exec_arguments(merged_environment)
+ exact_secret_values = _sensitive_environment_values(merged_environment)
+ exact_secret_values.update(_credential_uri_environment_values(merged_environment))
+
+ exec_command = ["exec"]
+ effective_cwd = cwd or self.task_env_config.workdir
+ if effective_cwd:
+ exec_command.extend(["-w", effective_cwd])
+ exec_command.extend(environment_args)
+ if user is not None:
+ exec_command.extend(["-u", str(user)])
+ exec_command.append("main")
+ exec_command.extend(self._platform.exec_shell_args(command))
+
+ return await self._run_docker_compose_command(
+ exec_command,
+ check=False,
+ timeout_sec=timeout_sec,
+ on_output=self._output_callback(),
+ env_overrides=subprocess_environment,
+ stop_main_on_interrupt=True,
+ **({"exact_secret_values": exact_secret_values} if exact_secret_values else {}),
+ )
+
+ def _main_only_compose_environment(self) -> tuple[set[str], set[str]]:
+ """Return main-only names and values that a sidecar client must not inherit."""
+ main_names: set[str] = set()
+ main_values: set[str] = set()
+
+ def include(environment: Mapping[str, str]) -> None:
+ main_names.update(environment)
+ main_values.update(_eligible_secret_values(environment.values()))
+ main_values.update(_sensitive_environment_values(environment))
+ main_values.update(_credential_uri_environment_values(environment))
+
+ include(getattr(self, "_compose_task_env", {}))
+ include(getattr(self, "_persistent_env", {}))
+ for scoped_environment in self._exec_env_overlays.get():
+ include(scoped_environment)
+
+ active_handoff_scopes = _SECURE_HANDOFF_SCOPES.get()
+ for handoff_scope in active_handoff_scopes:
+ main_names.update(handoff_scope.environment_names)
+ main_values.update(
+ _eligible_secret_values(
+ handoff_scope.secret_values,
+ include_short=True,
+ )
+ )
+
+ sentinel_names = {
+ "NVIDIA_API_KEY",
+ NVIDIA_BUILD_KEY_STDIN_ENV,
+ _NVIDIA_BUILD_KEY_FILE_ENV,
+ }
+ for name in sentinel_names:
+ value = os.environ.get(name)
+ if value is not None:
+ main_values.update(_eligible_secret_values((value,)))
+ if name != NVIDIA_BUILD_KEY_STDIN_ENV:
+ main_values.update(_eligible_secret_values((value,), include_short=True))
+
+ main_values.update(
+ _eligible_secret_values(
+ (
+ NVIDIA_BUILD_STDIN_SENTINEL,
+ _NVIDIA_BUILD_FILE_SENTINEL,
+ )
+ )
+ )
+ return main_names | sentinel_names, main_values
+
+ def _sidecar_exec_lock(self, service: str) -> asyncio.Lock:
+ locks = getattr(self, "_skillevaluator_sidecar_exec_locks", None)
+ if locks is None:
+ locks = {}
+ self._skillevaluator_sidecar_exec_locks = locks
+ lock = locks.get(service)
+ if lock is None:
+ lock = asyncio.Lock()
+ locks[service] = lock
+ return lock
+
+ def _compose_model_metadata(self) -> tuple[set[str], set[str]]:
+ """Inspect Compose interpolation and service keys with one hardened walk.
+
+ Compose's own ``config --no-interpolate`` rejects valid unresolved
+ values in typed fields (for example ``ports[].host_ip``), so it cannot
+ safely serve as metadata discovery. Parse YAML values instead and fail
+ closed for dynamic or external include/extends inputs.
+ """
+ environment_root = self.environment_dir.resolve()
+ try:
+ (environment_root / ".env").lstat()
+ except FileNotFoundError:
+ pass
+ except OSError as exc:
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs") from exc
+ else:
+ raise RuntimeError(
+ "Docker Compose project .env files are not supported; "
+ "declare interpolation values through the Harbor task environment"
+ )
+ root_paths = [path.resolve() for path in self._docker_compose_paths]
+ pending_models = [(path, environment_root) for path in root_paths]
+ trusted_roots = {environment_root, *(path.parent for path in root_paths)}
+ inspected_models: set[tuple[Path, Path]] = set()
+ interpolation_names: set[str] = set()
+ declared_services: set[str] = set()
+ total_bytes = 0
+
+ def is_trusted_path(path: Path) -> bool:
+ return any(path.is_relative_to(root) for root in trusted_roots)
+
+ def resolve_model_path(raw_path: object, *, relative_to: Path) -> Path:
+ if not isinstance(raw_path, str) or "$" in raw_path:
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ referenced_path = (relative_to / raw_path).resolve()
+ if not is_trusted_path(referenced_path):
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ return referenced_path
+
+ def reject_include_dotenv(project_directory: Path) -> None:
+ if (project_directory / ".env").exists():
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+
+ while pending_models:
+ path, project_directory = pending_models.pop()
+ model_identity = (path, project_directory)
+ if model_identity in inspected_models:
+ continue
+ if not is_trusted_path(path):
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ inspected_models.add(model_identity)
+ if len(inspected_models) > _MAX_COMPOSE_MODEL_FILES:
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ try:
+ flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)
+ if hasattr(os, "O_CLOEXEC"):
+ flags |= os.O_CLOEXEC
+ if hasattr(os, "O_NOFOLLOW"):
+ flags |= os.O_NOFOLLOW
+ descriptor = os.open(path, flags)
+ with os.fdopen(descriptor, "rb") as compose_file:
+ if not stat.S_ISREG(os.fstat(compose_file.fileno()).st_mode):
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ remaining_bytes = _MAX_COMPOSE_MODEL_BYTES - total_bytes
+ content = compose_file.read(remaining_bytes + 1)
+ if len(content) > remaining_bytes:
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ total_bytes += len(content)
+ # _ComposeModelLoader subclasses yaml.SafeLoader and registers only
+ # scalar, sequence, and mapping constructors for Compose's !reset and
+ # !override tags, so arbitrary Python-object construction stays disabled.
+ # Depth and node-count bounds are enforced separately after parsing.
+ model = yaml.load( # nosec B506
+ content.decode("utf-8"),
+ Loader=_ComposeModelLoader,
+ )
+ except (OSError, RecursionError, UnicodeDecodeError, yaml.YAMLError) as exc:
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs") from exc
+ if not isinstance(model, Mapping):
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+
+ values = [(value, 1) for value in model.values()]
+ visited_containers: set[int] = {id(model)}
+ inspected_nodes = 0
+ while values:
+ value, depth = values.pop()
+ inspected_nodes += 1
+ if inspected_nodes > _MAX_COMPOSE_MODEL_NODES or depth > _MAX_COMPOSE_MODEL_DEPTH:
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ if isinstance(value, str):
+ interpolation_names.update(_compose_interpolation_names(value))
+ elif isinstance(value, Mapping):
+ if id(value) in visited_containers:
+ continue
+ visited_containers.add(id(value))
+ # Compose interpolates YAML values, not mapping keys.
+ values.extend((nested_value, depth + 1) for nested_value in value.values())
+ elif isinstance(value, list):
+ if id(value) in visited_containers:
+ continue
+ visited_containers.add(id(value))
+ values.extend((nested_value, depth + 1) for nested_value in value)
+
+ includes = model.get("include", [])
+ if not isinstance(includes, list):
+ includes = [includes]
+ for include in includes:
+ if isinstance(include, str):
+ include_path = resolve_model_path(
+ include,
+ relative_to=project_directory,
+ )
+ reject_include_dotenv(include_path.parent)
+ pending_models.append((include_path, include_path.parent))
+ continue
+ if (
+ not isinstance(include, Mapping)
+ or "env_file" in include
+ or include.get("project_directory") is not None
+ ):
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ include_paths = include.get("path")
+ if not isinstance(include_paths, list):
+ include_paths = [include_paths]
+ resolved_include_paths = [
+ resolve_model_path(
+ include_path,
+ relative_to=project_directory,
+ )
+ for include_path in include_paths
+ ]
+ if not resolved_include_paths:
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ included_project_directory = resolved_include_paths[0].parent
+ reject_include_dotenv(included_project_directory)
+ pending_models.extend(
+ (include_path, included_project_directory) for include_path in resolved_include_paths
+ )
+
+ services = model.get("services", {})
+ if not isinstance(services, Mapping):
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ for service_name, service in services.items():
+ if not isinstance(service_name, str):
+ raise RuntimeError("could not inspect Docker Compose interpolation inputs")
+ declared_services.add(_validate_compose_service_name(service_name))
+ if not isinstance(service, Mapping):
+ continue
+ extends = service.get("extends")
+ if not isinstance(extends, Mapping) or "file" not in extends:
+ continue
+ extends_path = resolve_model_path(
+ extends["file"],
+ relative_to=project_directory,
+ )
+ pending_models.append((extends_path, project_directory))
+
+ return interpolation_names, declared_services
+
+ def _compose_model_interpolation_names(self) -> set[str]:
+ return self._compose_model_metadata()[0]
+
+ def _declared_compose_service_names(self) -> set[str]:
+ return self._compose_model_metadata()[1]
+
+ async def _required_compose_model_environment_names(self) -> set[str]:
+ """Return available, non-infrastructure names required by the model."""
+ available_names = set(self._compose_env_vars(include_os_env=True))
+ possible_names = self._compose_model_interpolation_names()
+ infrastructure_names = set(self._compose_infra_env_vars())
+ if self._windows_container_name:
+ infrastructure_names.add("HARBOR_CONTAINER_NAME")
+ return {name for name in possible_names & available_names if name not in infrastructure_names}
+
+ def _protected_main_environment_values(self) -> set[str]:
+ protected_values = {
+ NVIDIA_BUILD_STDIN_SENTINEL,
+ _NVIDIA_BUILD_FILE_SENTINEL,
+ }
+ exact_protected_values: set[str] = set()
+
+ def include(environment: Mapping[str, str]) -> None:
+ exact_protected_values.update(
+ value
+ for name, value in environment.items()
+ if name != NVIDIA_BUILD_KEY_STDIN_ENV and value and _SENSITIVE_ENV_NAME_RE.search(name)
+ )
+
+ include(os.environ)
+ include(getattr(self, "_compose_task_env", {}))
+ include(getattr(self, "_persistent_env", {}))
+ for scoped_environment in self._exec_env_overlays.get():
+ include(scoped_environment)
+ for name in (
+ "NVIDIA_API_KEY",
+ NVIDIA_BUILD_KEY_STDIN_ENV,
+ _NVIDIA_BUILD_KEY_FILE_ENV,
+ ):
+ value = os.environ.get(name)
+ if value:
+ protected_values.add(value)
+ if name != NVIDIA_BUILD_KEY_STDIN_ENV:
+ exact_protected_values.add(value)
+ return {
+ *_eligible_secret_values(protected_values),
+ *exact_protected_values,
+ }
+
+ def _other_main_environment_values(
+ self,
+ retained_name: str,
+ retained_value: str,
+ ) -> set[str]:
+ protected_values: set[str] = set()
+
+ def include(environment: Mapping[str, str]) -> None:
+ for name, value in environment.items():
+ if not value or (name == retained_name and value == retained_value):
+ continue
+ if len(value) >= _MIN_EXACT_SECRET_LENGTH or _SENSITIVE_ENV_NAME_RE.search(name):
+ protected_values.add(value)
+
+ include(getattr(self, "_compose_task_env", {}))
+ include(getattr(self, "_persistent_env", {}))
+ for scoped_environment in self._exec_env_overlays.get():
+ include(scoped_environment)
+ return set(
+ _eligible_secret_values(
+ protected_values,
+ include_short=True,
+ )
+ )
+
+ async def _sidecar_compose_model_environment(self) -> dict[str, str]:
+ """Retain only non-sensitive task values structurally required by Compose."""
+ possible_names = await self._required_compose_model_environment_names()
+ if not possible_names:
+ return {}
+
+ required_names = possible_names
+ compose_environment = _validate_environment(self._compose_env_vars(include_os_env=True))
+ protected_values = self._protected_main_environment_values()
+ retained_environment: dict[str, str] = {}
+ for name in sorted(required_names):
+ value = compose_environment[name]
+ if _SENSITIVE_ENV_NAME_RE.search(name):
+ raise RuntimeError(f"Docker Compose interpolation variable {name!r} requires protected execution state")
+ if self._is_compose_client_operational_name(name):
+ raise RuntimeError(f"Docker Compose interpolation variable {name!r} cannot use host client controls")
+ if any(
+ _value_contains_protected_value(value, protected_value)
+ for protected_value in (protected_values | self._other_main_environment_values(name, value))
+ if protected_value
+ ):
+ raise RuntimeError(f"Docker Compose interpolation variable {name!r} requires protected execution state")
+ retained_environment[name] = value
+ return retained_environment
+
+ @contextlib.asynccontextmanager
+ async def _sidecar_operation(
+ self,
+ service: str,
+ *,
+ discover_compose_model: bool = True,
+ ) -> AsyncIterator[None]:
+ """Serialize a service while rejecting callback reentry before waiting."""
+ service = _validate_compose_service_name(service)
+ active_operations = [operation for operation in _SIDECAR_EXEC_OPERATIONS.get() if operation.active]
+ if any(
+ operation.environment_identity == id(self) and operation.service == service
+ for operation in active_operations
+ ):
+ raise RuntimeError(f"reentrant sidecar operation for service {service!r} is not supported")
+ operation_key = (id(self), service)
+ if active_operations and operation_key <= max(
+ (operation.environment_identity, operation.service) for operation in active_operations
+ ):
+ raise RuntimeError("nested sidecar operation violates deterministic lock ordering")
+
+ async with self._sidecar_exec_lock(service):
+ compose_model_environment = (
+ await self._sidecar_compose_model_environment() if discover_compose_model else {}
+ )
+ operation = _SidecarOperation(
+ environment_identity=id(self),
+ service=service,
+ compose_model_environment=compose_model_environment,
+ )
+ token = _SIDECAR_EXEC_OPERATIONS.set((*_SIDECAR_EXEC_OPERATIONS.get(), operation))
+ try:
+ yield
+ finally:
+ operation.active = False
+ _SIDECAR_EXEC_OPERATIONS.reset(token)
+
+ @contextlib.contextmanager
+ def _compose_environment_scrub_scope(
+ self,
+ environment_names: Iterable[str],
+ secret_values: Iterable[str],
+ ) -> Iterator[None]:
+ scope = _SecureHandoffScope(
+ environment_names=frozenset(environment_names),
+ secret_values=frozenset(_eligible_secret_values(secret_values, include_short=True)),
+ )
+ token = _SECURE_HANDOFF_SCOPES.set((*_SECURE_HANDOFF_SCOPES.get(), scope))
+ try:
+ yield
+ finally:
+ _SECURE_HANDOFF_SCOPES.reset(token)
+
+ async def stop_service(self, service: str) -> None:
+ service = _validate_compose_service_name(service)
+ async with self._sidecar_operation(
+ service,
+ discover_compose_model=False,
+ ):
+ if service not in self._declared_compose_service_names():
+ raise RuntimeError(f"unknown Docker Compose service {service!r}")
+ with _raw_lifecycle_deadline_scope():
+ await self._stop_raw_service_containers(
+ service,
+ remove=False,
+ require_existing=False,
+ )
+ async def service_download_file(
+ self,
+ source_path: str,
+ target_path: Path | str,
+ *,
+ service: str | None = None,
+ ) -> None:
+ service = MAIN_SERVICE_NAME if service is None else service
+ service = _validate_compose_service_name(service)
+ excluded_names, excluded_values = self._main_only_compose_environment()
-def _redact_result(result: ExecResult, secret_values: set[str]) -> ExecResult:
- return ExecResult(
- stdout=_redact(result.stdout, secret_values),
- stderr=_redact(result.stderr, secret_values),
- return_code=result.return_code,
- )
+ if service == MAIN_SERVICE_NAME and self._is_windows_container:
+ target = Path(target_path)
+ parent_usage = _missing_windows_target_parent_usage(target)
+ final_reserve = _windows_artifact_filesystem_reserve(target.parent)
+ with tempfile.TemporaryDirectory() as temporary_directory:
+ source_parent, source_name = _split_windows_container_file_source(source_path)
+ await self._secure_windows_download_dir(
+ source_parent,
+ temporary_directory,
+ excluded_names=excluded_names,
+ excluded_values=excluded_values,
+ archive_members=(source_name,),
+ )
+ downloaded = Path(temporary_directory) / source_name
+ try:
+ downloaded_metadata = downloaded.lstat()
+ except OSError:
+ raise RuntimeError("requested file was not present in the Windows container download") from None
+ if (
+ stat_is_link_or_reparse(downloaded_metadata)
+ or not stat.S_ISREG(downloaded_metadata.st_mode)
+ or downloaded_metadata.st_nlink != 1
+ ):
+ raise RuntimeError("requested Windows container download was not a regular file")
+ source_usage = _WindowsArtifactUsage(
+ file_bytes=downloaded_metadata.st_size,
+ entries=1,
+ )
+ existing_target_usage = _existing_windows_target_usage(
+ target,
+ kind="file",
+ )
+ required_entries = source_usage.entries + existing_target_usage.entries + parent_usage.entries
+ _require_windows_artifact_resources(
+ target.parent,
+ source_usage.estimated_disk_bytes
+ + existing_target_usage.estimated_disk_bytes
+ + parent_usage.estimated_disk_bytes,
+ required_entries=required_entries,
+ minimum_free_bytes=final_reserve.minimum_free_bytes,
+ purpose="file publication",
+ )
+ try:
+ copy_file_secure(
+ downloaded,
+ target,
+ allowed_root=downloaded.parent,
+ )
+ except (OSError, ValueError):
+ raise RuntimeError("requested Windows container download could not be copied safely") from None
+ return
+ async def download() -> None:
+ if service == MAIN_SERVICE_NAME:
+ await self._run_docker_compose_command(
+ ["cp", "--", f"{service}:{source_path}", str(target_path)],
+ check=True,
+ additional_secret_values=excluded_values,
+ compose_env_excluded_names=excluded_names,
+ compose_env_excluded_values=excluded_values,
+ use_sidecar_compose_model=True,
+ )
+ return
+ self._sidecar_platform(service)
+ await self._run_docker_compose_command(
+ ["cp", "--", f"{service}:{source_path}", str(target_path)],
+ check=True,
+ additional_secret_values=excluded_values,
+ compose_env_excluded_names=excluded_names,
+ compose_env_excluded_values=excluded_values,
+ use_sidecar_compose_model=True,
+ )
-def _signal_process_tree(process: asyncio.subprocess.Process, value: signal.Signals) -> None:
- if process.returncode is not None:
- return
- if os.name == "posix":
- with contextlib.suppress(ProcessLookupError):
- os.killpg(process.pid, value)
- elif value == signal.SIGTERM:
- process.terminate()
- else:
- process.kill()
+ async with self._sidecar_operation(service):
+ await download()
+ async def service_download_dir(
+ self,
+ source_dir: str,
+ target_dir: Path | str,
+ *,
+ service: str | None = None,
+ ) -> None:
+ service = MAIN_SERVICE_NAME if service is None else service
+ service = _validate_compose_service_name(service)
+ excluded_names, excluded_values = self._main_only_compose_environment()
-async def _terminate_process_tree(
- process: asyncio.subprocess.Process,
- communication: asyncio.Task[tuple[bytes, bytes]],
- *,
- preserve_cancellation: bool,
-) -> None:
- async def reap() -> None:
- _signal_process_tree(process, signal.SIGTERM)
- try:
- await asyncio.wait_for(asyncio.shield(communication), timeout=_COMPOSE_TERMINATE_SECONDS)
+ if service == MAIN_SERVICE_NAME and self._is_windows_container:
+ await self._secure_windows_download_dir(
+ source_dir,
+ target_dir,
+ excluded_names=excluded_names,
+ excluded_values=excluded_values,
+ )
return
- except TimeoutError:
- pass
- _signal_process_tree(process, signal.SIGKILL)
+ async def download() -> None:
+ if service == MAIN_SERVICE_NAME:
+ await self._run_docker_compose_command(
+ ["cp", "--", f"{service}:{source_dir}/.", str(target_dir)],
+ check=True,
+ additional_secret_values=excluded_values,
+ compose_env_excluded_names=excluded_names,
+ compose_env_excluded_values=excluded_values,
+ use_sidecar_compose_model=True,
+ )
+ return
+ self._sidecar_platform(service)
+ await self._run_docker_compose_command(
+ ["cp", "--", f"{service}:{source_dir}/.", str(target_dir)],
+ check=True,
+ additional_secret_values=excluded_values,
+ compose_env_excluded_names=excluded_names,
+ compose_env_excluded_values=excluded_values,
+ use_sidecar_compose_model=True,
+ )
+
+ async with self._sidecar_operation(service):
+ await download()
+
+ async def _run_trusted_transfer_command(
+ self,
+ command: list[str],
+ *,
+ process_environment: Mapping[str, str],
+ protected_values: set[str],
+ output_path: Path,
+ idle_timeout_sec: float = _WINDOWS_TRANSFER_IDLE_TIMEOUT_SECONDS,
+ total_timeout_sec: float = _WINDOWS_TRANSFER_TOTAL_TIMEOUT_SECONDS,
+ minimum_free_bytes: int | None = None,
+ ) -> None:
+ """Stream untrusted output to disk with size, idle, and total bounds."""
+ if not math.isfinite(idle_timeout_sec) or idle_timeout_sec <= 0:
+ raise ValueError("idle_timeout_sec must be a positive finite value")
+ if not math.isfinite(total_timeout_sec) or total_timeout_sec <= 0:
+ raise ValueError("total_timeout_sec must be a positive finite value")
+ if minimum_free_bytes is not None and minimum_free_bytes < 0:
+ raise ValueError("minimum_free_bytes must not be negative")
+ executable = shutil.which(command[0], path=process_environment.get("PATH"))
+ if executable is None:
+ raise RuntimeError(f"required host transfer executable {command[0]!r} was not found")
+ initial_free_bytes, spool_budget = _windows_artifact_disk_budget(output_path.parent)
+ reserved_bytes = initial_free_bytes - spool_budget if minimum_free_bytes is None else minimum_free_bytes
+ spool_budget = max(0, initial_free_bytes - reserved_bytes)
+ if spool_budget <= 0:
+ raise RuntimeError("insufficient temporary disk space for secure Windows container transfer")
+ diagnostic_protected_values = set(protected_values)
+ diagnostic_protected_values.update(_sensitive_environment_values(process_environment))
+ diagnostic_protected_values.update(_credential_uri_environment_values(process_environment))
+ full_command = [executable, *command[1:]]
+ loop = asyncio.get_running_loop()
+ total_deadline = loop.time() + total_timeout_sec
+ output_created = False
try:
- await asyncio.wait_for(asyncio.shield(communication), timeout=_COMPOSE_KILL_SECONDS)
- except TimeoutError:
- communication.cancel()
- done, _pending = await asyncio.wait({communication}, timeout=_COMPOSE_CANCEL_SECONDS)
- if communication in done:
- with contextlib.suppress(asyncio.CancelledError, Exception):
- communication.result()
+ stdout_handle = output_path.open("xb", buffering=0)
+ output_created = True
+ with (
+ stdout_handle as stdout_file,
+ tempfile.TemporaryFile(
+ mode="w+b",
+ buffering=0,
+ dir=output_path.parent,
+ ) as stderr_file,
+ ):
+ creation = asyncio.create_task(
+ asyncio.create_subprocess_exec(
+ *full_command,
+ env=dict(process_environment),
+ stdin=asyncio.subprocess.DEVNULL,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ start_new_session=os.name == "posix",
+ )
+ )
+ try:
+ creation_timeout = min(
+ _RAW_DOCKER_COMMAND_TIMEOUT_SECONDS,
+ total_deadline - loop.time(),
+ )
+ if creation_timeout <= 0:
+ raise _WindowsTransferTotalTimeout
+ process = await asyncio.wait_for(
+ asyncio.shield(creation),
+ timeout=creation_timeout,
+ )
+ except (
+ TimeoutError,
+ asyncio.CancelledError,
+ _WindowsTransferTotalTimeout,
+ ) as primary_error:
- cleanup = asyncio.create_task(reap())
- await _await_task_uninterruptibly(cleanup, preserve_cancellation=preserve_cancellation)
+ async def reap_late_creation(
+ completed_creation: asyncio.Task[asyncio.subprocess.Process],
+ ) -> None:
+ try:
+ late_process = completed_creation.result()
+ except BaseException:
+ return
+ late_wait = asyncio.create_task(late_process.wait())
+ await _terminate_process_tree(
+ late_process,
+ late_wait,
+ preserve_cancellation=False,
+ )
+ def schedule_late_reap(
+ completed_creation: asyncio.Task[asyncio.subprocess.Process],
+ ) -> None:
+ late_cleanup = asyncio.create_task(reap_late_creation(completed_creation))
-def _host_handoff_environment(environment: Mapping[str, str]) -> dict[str, str]:
- """Resolve a private NVIDIA Build sentinel without putting its value in argv."""
- resolved = _validate_environment(environment)
- if resolved.get("NVIDIA_API_KEY") == NVIDIA_BUILD_STDIN_SENTINEL:
- resolved["NVIDIA_API_KEY"] = read_nvidia_build_key_from_stdin()
- return resolved
- if resolved.get("NVIDIA_API_KEY") != _NVIDIA_BUILD_FILE_SENTINEL:
- return resolved
- key_file = os.environ.get(_NVIDIA_BUILD_KEY_FILE_ENV, "").strip()
- if not key_file:
- raise RuntimeError(f"{_NVIDIA_BUILD_KEY_FILE_ENV} is required for NVIDIA Build Docker runs")
- try:
- api_key = Path(key_file).read_text(encoding="utf-8").strip()
- except OSError as exc:
- raise RuntimeError("NVIDIA Build key handoff file is unavailable") from exc
- if not api_key:
- raise RuntimeError("NVIDIA Build key handoff file is empty")
- resolved["NVIDIA_API_KEY"] = api_key
- return resolved
+ def report_late_cleanup_failure(
+ completed_cleanup: asyncio.Task[None],
+ ) -> None:
+ try:
+ completed_cleanup.result()
+ except BaseException as exc:
+ asyncio.get_running_loop().call_exception_handler(
+ {
+ "message": "late Windows transfer client cleanup failed",
+ "exception": exc,
+ "task": completed_cleanup,
+ }
+ )
+ late_cleanup.add_done_callback(report_late_cleanup_failure)
-def _render_environment_script(environment: Mapping[str, str]) -> str:
- """Render a sourceable script after validating every name and value."""
- validated = _validate_environment(environment)
- lines = [f"export {name}={shlex.quote(value)}" for name, value in sorted(validated.items())]
- return "\n".join(lines) + "\n"
+ async def cancel_creation_race() -> tuple[
+ asyncio.subprocess.Process | None,
+ bool,
+ ]:
+ creation.cancel()
+ done, _pending = await asyncio.wait(
+ {creation},
+ timeout=_COMPOSE_CANCEL_SECONDS,
+ )
+ if creation not in done:
+ creation.add_done_callback(schedule_late_reap)
+ return None, False
+ try:
+ return creation.result(), True
+ except BaseException:
+ return None, True
+ cancellation = asyncio.create_task(cancel_creation_race())
+ process, creation_resolved = await _await_task_uninterruptibly(
+ cancellation,
+ preserve_cancellation=False,
+ )
+ if process is not None:
+ process_wait = asyncio.create_task(process.wait())
+ await _terminate_process_tree(
+ process,
+ process_wait,
+ preserve_cancellation=False,
+ )
+ if not creation_resolved:
+ primary_error.add_note(
+ "Windows transfer process creation cancellation remained pending; "
+ "a late-process reaper was installed"
+ )
+ if isinstance(primary_error, TimeoutError):
+ raise RuntimeError(
+ "secure Windows container transfer process creation timed out"
+ ) from primary_error
+ if isinstance(primary_error, _WindowsTransferTotalTimeout):
+ raise RuntimeError(
+ "secure Windows container transfer command exceeded its total time limit"
+ ) from primary_error
+ raise
-class SkillEvaluatorDockerEnvironment(DockerEnvironment):
- """Pinned Harbor compatibility backend with host-visible argv safety."""
+ process_wait = asyncio.create_task(process.wait())
+ pump_tasks: set[asyncio.Task[None]] = set()
+ try:
+ stdout_stream = process.stdout
+ stderr_stream = process.stderr
+ if stdout_stream is None or stderr_stream is None:
+ raise RuntimeError("secure Windows container transfer pipes were unavailable")
- @classmethod
- def preflight(cls) -> None:
- """Consume the private stdin handoff before Docker can inherit it."""
- if os.environ.get("NVIDIA_API_KEY", "").strip() == NVIDIA_BUILD_STDIN_SENTINEL:
- read_nvidia_build_key_from_stdin()
- super().preflight()
+ spooled_bytes = 0
- async def _contain_main_container(self) -> None:
- """Stop and remove this Compose project's task container from the trusted host."""
- stopped = False
- try:
- result = await self._run_docker_compose_command(
- ["stop", "--timeout", "0", "main"],
- check=False,
- timeout_sec=_MAIN_CONTAINER_STOP_TIMEOUT_SECONDS,
- )
- stopped = result.return_code == 0
- except Exception:
- pass
+ async def pump_stream(
+ stream: asyncio.StreamReader,
+ destination: BinaryIO,
+ ) -> None:
+ nonlocal spooled_bytes
+ while chunk := await stream.read(64 * 1024):
+ claimed_bytes = spooled_bytes + len(chunk)
+ free_bytes = shutil.disk_usage(output_path.parent).free
+ if claimed_bytes > spool_budget or free_bytes < reserved_bytes + len(chunk):
+ raise _WindowsTransferDiskBudgetExceeded
+ view = memoryview(chunk)
+ while view:
+ written = destination.write(view)
+ if written is None or written <= 0:
+ raise OSError("Windows transfer spool write made no progress")
+ view = view[written:]
+ spooled_bytes = claimed_bytes
- if not stopped:
- try:
- result = await self._run_docker_compose_command(
- ["kill", "--signal", "SIGKILL", "main"],
- check=False,
- timeout_sec=_MAIN_CONTAINER_STOP_TIMEOUT_SECONDS,
- )
- stopped = result.return_code == 0
- except Exception:
- pass
+ pump_tasks = {
+ asyncio.create_task(pump_stream(stdout_stream, stdout_file)),
+ asyncio.create_task(pump_stream(stderr_stream, stderr_file)),
+ }
+ idle_deadline = loop.time() + idle_timeout_sec
+ last_spooled_bytes = 0
+ process_complete = False
+ active_pumps = set(pump_tasks)
+ while True:
+ current_time = loop.time()
+ if current_time >= total_deadline:
+ raise _WindowsTransferTotalTimeout
+ remaining_idle = idle_deadline - current_time
+ if remaining_idle <= 0:
+ raise _ComposeCommandTimeout
+ watched: set[asyncio.Task[Any]] = set(active_pumps)
+ if not process_complete:
+ watched.add(process_wait)
+ done, _pending = await asyncio.wait(
+ watched,
+ return_when=asyncio.FIRST_COMPLETED,
+ timeout=min(
+ _WINDOWS_TRANSFER_POLL_SECONDS,
+ remaining_idle,
+ total_deadline - current_time,
+ ),
+ )
+ current_time = loop.time()
+ if current_time >= total_deadline:
+ raise _WindowsTransferTotalTimeout
+ if process_wait in done:
+ process_wait.result()
+ process_complete = True
+ completed_pumps = active_pumps.intersection(done)
+ for completed_pump in completed_pumps:
+ completed_pump.result()
+ active_pumps.difference_update(completed_pumps)
+ if process_complete and not active_pumps:
+ break
+ if spooled_bytes != last_spooled_bytes:
+ last_spooled_bytes = spooled_bytes
+ idle_deadline = current_time + idle_timeout_sec
+ elif current_time >= idle_deadline:
+ raise _ComposeCommandTimeout
+ except BaseException as primary_error:
+ await _terminate_process_tree(
+ process,
+ process_wait,
+ preserve_cancellation=False,
+ )
+ for pump_task in pump_tasks:
+ if not pump_task.done():
+ pump_task.cancel()
- # Removal destroys a handoff that cancellation may have interrupted
- # before the in-container wrapper could unlink it. ``--stop`` is also
- # the final host-authoritative fallback if stop/kill was inconclusive.
- try:
- result = await self._run_docker_compose_command(
- ["rm", "--force", "--stop", "--volumes", "main"],
- check=False,
- timeout_sec=_MAIN_CONTAINER_STOP_TIMEOUT_SECONDS,
- )
- except Exception as exc:
- raise RuntimeError("could not confirm main task container containment") from exc
- if result.return_code != 0:
- detail = "after a confirmed stop" if stopped else "after inconclusive stop and kill attempts"
- raise RuntimeError(
- f"could not confirm main task container containment (removal status {result.return_code} {detail})"
- )
+ async def finish_pumps() -> None:
+ if pump_tasks:
+ await asyncio.gather(*pump_tasks, return_exceptions=True)
- async def _contain_main_and_reap_compose(
+ pump_cleanup = asyncio.create_task(finish_pumps())
+ await _await_task_uninterruptibly(
+ pump_cleanup,
+ preserve_cancellation=False,
+ )
+ if isinstance(primary_error, _ComposeCommandTimeout):
+ raise RuntimeError(
+ "secure Windows container transfer command timed out while idle"
+ ) from primary_error
+ if isinstance(primary_error, _WindowsTransferDiskBudgetExceeded):
+ raise RuntimeError(
+ "secure Windows container transfer exceeded its temporary disk budget"
+ ) from primary_error
+ if isinstance(primary_error, _WindowsTransferTotalTimeout):
+ raise RuntimeError(
+ "secure Windows container transfer command exceeded its total time limit"
+ ) from primary_error
+ raise
+
+ if process.returncode != 0:
+ stderr_size = os.fstat(stderr_file.fileno()).st_size
+ eligible_secrets = _eligible_secret_values(
+ diagnostic_protected_values,
+ include_short=True,
+ )
+ maximum_secret_bytes = max(
+ (len(secret.encode("utf-8", errors="surrogatepass")) for secret in eligible_secrets),
+ default=0,
+ )
+ if maximum_secret_bytes > _WINDOWS_TRANSFER_STDERR_MAX_BYTES or (
+ eligible_secrets and stderr_size > _WINDOWS_TRANSFER_STDERR_MAX_BYTES
+ ):
+ detail = (
+ "[transfer diagnostics omitted because protected values "
+ "cannot be redacted safely after truncation]"
+ )
+ else:
+ read_limit = _WINDOWS_TRANSFER_STDERR_MAX_BYTES
+ stderr_file.seek(max(0, stderr_size - read_limit))
+ stderr = stderr_file.read(read_limit)
+ detail = _redact(
+ stderr.decode(errors="replace"),
+ diagnostic_protected_values,
+ include_short=True,
+ )
+ truncated = stderr_size > len(stderr) or len(detail) > _WINDOWS_TRANSFER_STDERR_MAX_BYTES
+ detail = detail[-_WINDOWS_TRANSFER_STDERR_MAX_BYTES:]
+ if truncated:
+ detail = "[earlier transfer diagnostics truncated]\n" + detail
+ raise RuntimeError("secure Windows container transfer command failed: " + detail)
+ except BaseException:
+ if output_created:
+ with contextlib.suppress(OSError):
+ output_path.unlink()
+ raise
+
+ async def _secure_windows_download_dir(
self,
- process: asyncio.subprocess.Process,
- communication: asyncio.Task[tuple[bytes, bytes]],
+ source_dir: str,
+ target_dir: Path | str,
*,
- stop_main_on_interrupt: bool,
+ excluded_names: set[str],
+ excluded_values: set[str],
+ archive_members: tuple[str, ...] = (".",),
) -> None:
- containment_error: BaseException | None = None
- if stop_main_on_interrupt:
- try:
- await self._contain_main_container()
- except BaseException as exc:
- containment_error = exc
- await _terminate_process_tree(process, communication, preserve_cancellation=False)
- if containment_error is not None:
- raise RuntimeError("could not confirm main task container containment") from containment_error
+ container_name = self._windows_container_name
+ if not container_name or not _COMPOSE_SERVICE_NAME_RE.fullmatch(container_name):
+ raise RuntimeError("Windows container transfer target is invalid")
+ process_environment = self._trusted_compose_client_environment(
+ excluded_names=excluded_names,
+ excluded_values=excluded_values,
+ )
+ target = Path(target_dir)
+ parent_usage = _missing_windows_target_parent_usage(target)
+ target_reserve = _windows_artifact_filesystem_reserve(target.parent)
+ with tempfile.TemporaryDirectory(prefix="skillevaluator-windows-download-") as temporary_directory:
+ private_root = Path(temporary_directory)
+ temporary_reserve = _windows_artifact_filesystem_reserve(private_root)
+ if temporary_reserve.identity == target_reserve.identity:
+ common_reserve = max(
+ temporary_reserve.minimum_free_bytes,
+ target_reserve.minimum_free_bytes,
+ )
+ temporary_reserve = _WindowsFilesystemReserve(
+ identity=temporary_reserve.identity,
+ minimum_free_bytes=common_reserve,
+ )
+ target_reserve = _WindowsFilesystemReserve(
+ identity=target_reserve.identity,
+ minimum_free_bytes=common_reserve,
+ )
+ archive_path = private_root / "container.tar"
+ extracted = private_root / "extracted"
+ extracted.mkdir(mode=0o700)
+ await self._run_trusted_transfer_command(
+ [
+ "docker",
+ "exec",
+ "--",
+ container_name,
+ "tar",
+ "cf",
+ "-",
+ "-C",
+ source_dir,
+ "--",
+ *archive_members,
+ ],
+ process_environment=process_environment,
+ protected_values=excluded_values,
+ output_path=archive_path,
+ minimum_free_bytes=temporary_reserve.minimum_free_bytes,
+ )
+ usage = _extract_regular_tar_archive(
+ archive_path,
+ extracted,
+ minimum_free_bytes=temporary_reserve.minimum_free_bytes,
+ )
+ existing_target_usage = _existing_windows_target_usage(
+ target,
+ kind="directory",
+ )
+ # ``copytree_secure(..., dirs_exist_ok=True)`` builds one merged
+ # stage plus a rollback snapshot of an existing destination. The
+ # extracted source remains live until publication completes.
+ required_entries = usage.entries + 2 * existing_target_usage.entries + parent_usage.entries
+ _require_windows_artifact_resources(
+ target.parent,
+ usage.estimated_disk_bytes
+ + 2 * existing_target_usage.estimated_disk_bytes
+ + parent_usage.estimated_disk_bytes,
+ required_entries=required_entries,
+ minimum_free_bytes=target_reserve.minimum_free_bytes,
+ purpose="directory publication",
+ )
+ copytree_secure(
+ extracted,
+ target,
+ dirs_exist_ok=True,
+ allowed_root=extracted,
+ )
- async def exec(
+ async def service_exec(
self,
command: str,
+ *,
+ service: str | None = None,
cwd: str | None = None,
env: dict[str, str] | None = None,
timeout_sec: int | None = None,
user: str | int | None = None,
) -> ExecResult:
- user = self._resolve_user(user)
- merged_environment = self._merge_env(env)
- environment_args, subprocess_environment = _secure_exec_arguments(merged_environment)
+ """Execute in main securely, or in an isolated POSIX sidecar."""
+ if service is None or service == MAIN_SERVICE_NAME:
+ return await self.exec(
+ command,
+ cwd=cwd,
+ env=env,
+ timeout_sec=timeout_sec,
+ user=user,
+ )
+ if self._is_windows_container:
+ raise ServiceOperationsUnsupportedError(
+ f"Per-service operations are not supported for Windows containers (requested service: {service!r})."
+ )
+ service = _validate_compose_service_name(service)
+ return await self._secure_compose_exec(
+ command,
+ service=service,
+ cwd=cwd,
+ env=env,
+ timeout_sec=timeout_sec,
+ user=user,
+ )
- exec_command = ["exec"]
- effective_cwd = cwd or self.task_env_config.workdir
- if effective_cwd:
- exec_command.extend(["-w", effective_cwd])
- exec_command.extend(environment_args)
- if user is not None:
- exec_command.extend(["-u", str(user)])
- exec_command.append("main")
- exec_command.extend(self._platform.exec_shell_args(command))
+ async def _secure_compose_exec(
+ self,
+ command: str,
+ *,
+ service: str,
+ cwd: str | None,
+ env: dict[str, str] | None,
+ timeout_sec: int | None,
+ user: str | int | None,
+ ) -> ExecResult:
+ """Run a sidecar command without exposing values or main-only state."""
+ service = _validate_compose_service_name(service)
+ validated_environment = _validate_environment(env)
+ async with self._sidecar_operation(service):
+ reserved_names = set(self._compose_env_vars(include_os_env=True))
+ environment_args, carrier_environment, environment_wrapper = _sidecar_environment_carriers(
+ validated_environment,
+ reserved_names=reserved_names,
+ )
+ carrier_names = set(carrier_environment)
+ secret_values = set(_eligible_secret_values(validated_environment.values()))
+ exact_secret_values = _sensitive_environment_values(validated_environment)
+ exact_secret_values.update(_credential_uri_environment_values(validated_environment))
+ secret_values.update(exact_secret_values)
+ main_names, main_values = self._main_only_compose_environment()
+ with self._compose_environment_scrub_scope(
+ main_names | set(validated_environment) | carrier_names,
+ main_values | secret_values,
+ ):
+ excluded_names, excluded_values = self._main_only_compose_environment()
+ exec_command = ["exec"]
+ if cwd:
+ exec_command.extend(["-w", cwd])
+ exec_command.extend(environment_args)
+ if user is not None:
+ exec_command.extend(["-u", str(user)])
+ exec_command.extend(["--", service, "sh", "-c"])
+ if environment_wrapper is None:
+ exec_command.append(command)
+ else:
+ exec_command.extend([environment_wrapper, "sh", command])
- return await self._run_docker_compose_command(
- exec_command,
- check=False,
- timeout_sec=timeout_sec,
- env_overrides=subprocess_environment,
- stop_main_on_interrupt=True,
+ return await self._run_docker_compose_command(
+ exec_command,
+ check=False,
+ timeout_sec=timeout_sec,
+ on_output=self._output_callback(),
+ sidecar_env_carriers=carrier_environment,
+ additional_secret_values=excluded_values,
+ compose_env_excluded_names=excluded_names,
+ compose_env_excluded_values=excluded_values,
+ use_sidecar_compose_model=True,
+ contain_service_on_interrupt=service,
+ stop_main_on_interrupt=False,
+ **({"exact_secret_values": exact_secret_values} if exact_secret_values else {}),
+ )
+
+ @staticmethod
+ def _is_compose_client_operational_name(name: str) -> bool:
+ normalized_name = name.upper()
+ return (
+ normalized_name
+ in {
+ "PATH",
+ "HOME",
+ "USER",
+ "LOGNAME",
+ "TMPDIR",
+ "TMP",
+ "TEMP",
+ "XDG_CONFIG_HOME",
+ "XDG_RUNTIME_DIR",
+ "LANG",
+ "LANGUAGE",
+ "SSL_CERT_FILE",
+ "SSL_CERT_DIR",
+ "SYSTEMROOT",
+ "WINDIR",
+ "COMSPEC",
+ "PATHEXT",
+ "USERPROFILE",
+ "HOMEDRIVE",
+ "HOMEPATH",
+ "APPDATA",
+ "LOCALAPPDATA",
+ "PROGRAMDATA",
+ "SSH_AUTH_SOCK",
+ "SSH_AGENT_PID",
+ "DBUS_SESSION_BUS_ADDRESS",
+ "GNUPGHOME",
+ "TERM",
+ "COLORTERM",
+ "NO_COLOR",
+ }
+ or normalized_name.startswith(("DOCKER_", "COMPOSE_", "LC_"))
+ or normalized_name.endswith("_PROXY")
)
+ @classmethod
+ def _is_trusted_compose_client_host_name(cls, name: str) -> bool:
+ normalized_name = name.upper()
+ if normalized_name.startswith("COMPOSE_"):
+ return normalized_name in {
+ "COMPOSE_ANSI",
+ "COMPOSE_HTTP_TIMEOUT",
+ "COMPOSE_IGNORE_ORPHANS",
+ "COMPOSE_PARALLEL_LIMIT",
+ "COMPOSE_PROGRESS",
+ "COMPOSE_STATUS_STDOUT",
+ }
+ return cls._is_compose_client_operational_name(name)
+
+ def _trusted_compose_client_environment(
+ self,
+ *,
+ excluded_names: set[str],
+ excluded_values: set[str],
+ ) -> dict[str, str]:
+ """Build a host/Harbor baseline without main or sidecar target state."""
+ # Values originate exclusively from the host operational allowlist or
+ # Harbor-generated infrastructure. Incidental short caller-chosen byte
+ # overlap must not disable or replace those authoritative controls.
+ del excluded_names
+ infrastructure = self._compose_infra_env_vars()
+ trusted_environment = {
+ name: value for name, value in os.environ.items() if self._is_trusted_compose_client_host_name(name)
+ }
+ trusted_environment.update(infrastructure)
+ if self._windows_container_name:
+ trusted_environment["HARBOR_CONTAINER_NAME"] = self._windows_container_name
+
+ for name, value in trusted_environment.items():
+ if any(
+ value == protected_value
+ or (len(protected_value) >= _MIN_EXACT_SECRET_LENGTH and protected_value in value)
+ for protected_value in excluded_values
+ ):
+ raise RuntimeError(
+ f"trusted Docker Compose client environment variable {name!r} contains protected execution state"
+ )
+ return trusted_environment
+
async def _run_docker_compose_command(
self,
command: list[str],
check: bool = True,
- timeout_sec: int | None = None,
+ timeout_sec: float | None = None,
+ stdin_data: bytes | None = None,
+ on_output: OutputCallback | None = None,
*,
env_overrides: Mapping[str, str] | None = None,
- stdin_bytes: bytes | None = None,
- redact_values: set[str] | None = None,
+ sidecar_env_carriers: Mapping[str, str] | None = None,
+ additional_secret_values: Iterable[str] | None = None,
+ exact_secret_values: Iterable[str] | None = None,
+ compose_env_excluded_names: Iterable[str] | None = None,
+ compose_env_excluded_values: Iterable[str] | None = None,
+ use_sidecar_compose_model: bool = False,
+ contain_service_on_interrupt: str | None = None,
stop_main_on_interrupt: bool = False,
) -> ExecResult:
"""Run compose with sensitive values only in child env or stdin."""
+ if stop_main_on_interrupt and contain_service_on_interrupt is not None:
+ raise ValueError("only one interrupt-containment target may be configured")
+ if contain_service_on_interrupt is not None:
+ contain_service_on_interrupt = _validate_compose_service_name(contain_service_on_interrupt)
+ if contain_service_on_interrupt == MAIN_SERVICE_NAME:
+ raise ValueError("sidecar containment cannot target the main service")
+ containment_target = (
+ "main task container"
+ if stop_main_on_interrupt
+ else (
+ f"sidecar service {contain_service_on_interrupt!r}"
+ if contain_service_on_interrupt is not None
+ else "Docker Compose client"
+ )
+ )
+ docker_executable = shutil.which("docker") or "docker"
full_command = [
- "docker",
+ docker_executable,
"compose",
"--project-name",
_sanitize_docker_compose_project_name(self.session_id),
@@ -303,17 +3031,107 @@ async def _run_docker_compose_command(
full_command.extend(["-f", str(path.resolve().absolute())])
full_command.extend(command)
- process_environment = self._compose_env_vars(include_os_env=True)
- process_environment.update(env_overrides or {})
- secret_values = {
- value for value in (env_overrides or {}).values() if value and len(value) >= _MIN_EXACT_SECRET_LENGTH
+ active_handoff_scopes = _SECURE_HANDOFF_SCOPES.get()
+ effective_env_overrides = _validate_environment(env_overrides)
+ carrier_overrides = _validate_environment(sidecar_env_carriers)
+ if carrier_overrides and (
+ not active_handoff_scopes
+ or any(not name.startswith(_SIDECAR_ENV_CARRIER_PREFIX) for name in carrier_overrides)
+ ):
+ raise ValueError("sidecar environment carriers require an active secure sidecar scope")
+ active_handoff_names: set[str] = set()
+ active_handoff_values: set[str] = set()
+ for handoff_scope in active_handoff_scopes:
+ active_handoff_names.update(handoff_scope.environment_names)
+ active_handoff_values.update(handoff_scope.secret_values)
+ if active_handoff_scopes:
+ active_handoff_names.update(effective_env_overrides)
+ active_handoff_values.update(_eligible_secret_values(effective_env_overrides.values()))
+ active_handoff_names.update(carrier_overrides)
+ active_handoff_values.update(_eligible_secret_values(carrier_overrides.values()))
+
+ isolate_compose_base = compose_env_excluded_names is not None or bool(active_handoff_scopes)
+ compose_model_environment: dict[str, str] = {}
+ if isolate_compose_base:
+ active_operation = next(
+ (
+ operation
+ for operation in reversed(_SIDECAR_EXEC_OPERATIONS.get())
+ if operation.active and operation.environment_identity == id(self)
+ ),
+ None,
+ )
+ if active_operation is not None:
+ compose_model_environment = active_operation.compose_model_environment
+ elif use_sidecar_compose_model:
+ raise RuntimeError("sidecar Compose model requested outside a protected operation")
+ else:
+ compose_model_environment = await self._sidecar_compose_model_environment()
+ if isolate_compose_base:
+ excluded_names = set(compose_env_excluded_names or ()) | active_handoff_names
+ excluded_values = set(
+ _eligible_secret_values(
+ compose_env_excluded_values or (),
+ include_short=True,
+ )
+ )
+ excluded_values.update(active_handoff_values)
+ process_environment = self._trusted_compose_client_environment(
+ excluded_names=excluded_names,
+ excluded_values=excluded_values,
+ )
+ # Never let Compose auto-load project .env files or a host-provided
+ # COMPOSE_ENV_FILES path across this isolation boundary.
+ process_environment["COMPOSE_DISABLE_ENV_FILE"] = "1"
+ # Only collision-checked internal carriers may cross an active
+ # scrub scope. Arbitrary overrides remain scrubbed as in Task 4.
+ process_environment.update(compose_model_environment)
+ process_environment.update(carrier_overrides)
+ else:
+ process_environment = self._compose_env_vars(include_os_env=True)
+ process_environment.update(effective_env_overrides)
+
+ compose_client_environment = {
+ name: value for name, value in process_environment.items() if self._is_compose_client_operational_name(name)
}
- secret_values.update(redact_values or set())
+ secret_values = set(_eligible_secret_values(effective_env_overrides.values()))
+ secret_values.update(_eligible_secret_values(carrier_overrides.values()))
+ secret_values.update(_eligible_secret_values(compose_model_environment.values()))
+ secret_values.update(_compose_client_credential_values(compose_client_environment))
+ secret_values.update(_credential_uri_environment_values(effective_env_overrides))
+ secret_values.update(_credential_uri_environment_values(carrier_overrides))
+ secret_values.update(_credential_uri_environment_values(compose_model_environment))
+ secret_values.update(
+ _eligible_secret_values(
+ additional_secret_values or (),
+ include_short=True,
+ )
+ )
+ secret_values.update(active_handoff_values)
+ secret_values.update(_eligible_secret_values(exact_secret_values or (), include_short=True))
+ redaction_marker = _collision_safe_redaction_marker(
+ secret_values,
+ include_short=True,
+ )
+
+ async def discard_output(_text: str, _stream: OutputStream) -> None:
+ return None
+
+ async def collect_bounded_output(
+ active_process: asyncio.subprocess.Process,
+ ) -> ExecResult:
+ return await self._collect_streamed_output(
+ active_process,
+ timeout_sec=None,
+ stdin_data=stdin_data,
+ on_output=discard_output,
+ )
+
creation = asyncio.create_task(
asyncio.create_subprocess_exec(
*full_command,
env=process_environment,
- stdin=asyncio.subprocess.PIPE if stdin_bytes is not None else asyncio.subprocess.DEVNULL,
+ stdin=(asyncio.subprocess.PIPE if stdin_data is not None else asyncio.subprocess.DEVNULL),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
start_new_session=os.name == "posix",
@@ -321,81 +3139,189 @@ async def _run_docker_compose_command(
)
try:
process = await asyncio.shield(creation)
- except asyncio.CancelledError:
+ except asyncio.CancelledError as primary_error:
process = await _await_task_uninterruptibly(creation, preserve_cancellation=False)
- communication = asyncio.create_task(
- process.communicate(stdin_bytes) if stdin_bytes is not None else process.communicate()
- )
+ communication = asyncio.create_task(collect_bounded_output(process))
cleanup = asyncio.create_task(
self._contain_main_and_reap_compose(
process,
communication,
+ contain_service_on_interrupt=contain_service_on_interrupt,
stop_main_on_interrupt=stop_main_on_interrupt,
)
)
try:
await _await_task_uninterruptibly(cleanup, preserve_cancellation=False)
- except RuntimeError as exc:
- raise RuntimeError(
- "main task container containment could not be confirmed during cancellation"
- ) from exc
+ except BaseException as cleanup_error:
+ primary_error.add_note(
+ f"{containment_target} containment could not be confirmed during cancellation: "
+ f"{type(cleanup_error).__name__}: {cleanup_error}"
+ )
+ raise primary_error from cleanup_error
raise
- communication = asyncio.create_task(
- process.communicate(stdin_bytes) if stdin_bytes is not None else process.communicate()
- )
- try:
- if timeout_sec:
- stdout_bytes, stderr_bytes = await asyncio.wait_for(
- asyncio.shield(communication),
- timeout=timeout_sec,
+ callback_error: asyncio.Future[BaseException] | None = None
+
+ if on_output is None:
+ communication = asyncio.create_task(collect_bounded_output(process))
+ else:
+ callback_error = asyncio.get_running_loop().create_future()
+ stream_redactor = _StreamingSecretRedactor(
+ secret_values,
+ _replacement=redaction_marker,
+ _include_short=True,
+ )
+
+ async def emit_redacted_output(text: str, stream: OutputStream) -> None:
+ if callback_error.done():
+ await asyncio.sleep(0)
+ return
+ try:
+ await on_output(text, stream)
+ except BaseException as exc:
+ if not callback_error.done():
+ callback_error.set_result(exc)
+ # Do not re-raise into Harbor's collector: it would race its
+ # immediate-process termination against our process-group and
+ # optional main-container cleanup. The outer owner wakes on
+ # callback_error and performs containment exactly once.
+ await asyncio.sleep(0)
+
+ async def redacted_callback(text: str, stream: OutputStream) -> None:
+ if callback_error.done():
+ await asyncio.sleep(0)
+ return
+ redacted_text = stream_redactor.feed(text)
+ if redacted_text:
+ await emit_redacted_output(redacted_text, stream)
+
+ async def collect_streamed_output() -> ExecResult:
+ try:
+ result = await self._collect_streamed_output(
+ process,
+ timeout_sec=None,
+ stdin_data=stdin_data,
+ on_output=redacted_callback,
+ )
+ if not callback_error.done():
+ final_output = stream_redactor.finish()
+ if final_output:
+ await emit_redacted_output(final_output, "stdout")
+ return result
+ except BaseException as exc:
+ if not callback_error.done():
+ callback_error.set_result(exc)
+ # The outer owner performs process-group cleanup before it
+ # propagates the original collector/callback exception.
+ return ExecResult(
+ stdout=None,
+ stderr=None,
+ return_code=process.returncode or 0,
+ )
+
+ communication = asyncio.create_task(collect_streamed_output())
+
+ async def cleanup_preserving_primary(primary_error: BaseException) -> None:
+ cleanup = asyncio.create_task(
+ self._contain_main_and_reap_compose(
+ process,
+ communication,
+ contain_service_on_interrupt=contain_service_on_interrupt,
+ stop_main_on_interrupt=stop_main_on_interrupt,
)
+ )
+ try:
+ await _await_task_uninterruptibly(cleanup, preserve_cancellation=False)
+ except BaseException as cleanup_error:
+ primary_error.add_note(
+ "Docker Compose cleanup or container containment/restoration also failed: "
+ f"{type(cleanup_error).__name__}: {cleanup_error}"
+ )
+ raise primary_error from cleanup_error
+
+ callback_failure: BaseException | None = None
+ try:
+ waitables: set[asyncio.Future[Any] | asyncio.Task[Any]] = {communication}
+ if callback_error is not None:
+ waitables.add(callback_error)
+ done, _pending = await asyncio.wait(
+ waitables,
+ timeout=timeout_sec or None,
+ return_when=asyncio.FIRST_COMPLETED,
+ )
+ if not done:
+ raise _ComposeCommandTimeout
+ if callback_error is not None and callback_error in done:
+ callback_failure = callback_error.result()
else:
- stdout_bytes, stderr_bytes = await asyncio.shield(communication)
- except TimeoutError:
+ result = await asyncio.shield(communication)
+ except _ComposeCommandTimeout:
+ primary_error = RuntimeError(f"Command timed out after {timeout_sec} seconds")
cleanup = asyncio.create_task(
self._contain_main_and_reap_compose(
process,
communication,
+ contain_service_on_interrupt=contain_service_on_interrupt,
stop_main_on_interrupt=stop_main_on_interrupt,
)
)
try:
await _await_task_uninterruptibly(cleanup)
- except RuntimeError as exc:
- raise RuntimeError(
- f"Command timed out after {timeout_sec} seconds; main task container containment could not be confirmed"
- ) from exc
- raise RuntimeError(f"Command timed out after {timeout_sec} seconds") from None
- except asyncio.CancelledError:
+ except Exception as cleanup_error:
+ primary_error.add_note(
+ f"{containment_target} containment could not be confirmed: "
+ f"{type(cleanup_error).__name__}: {cleanup_error}"
+ )
+ raise primary_error from cleanup_error
+ raise primary_error from None
+ except asyncio.CancelledError as primary_error:
cleanup = asyncio.create_task(
self._contain_main_and_reap_compose(
process,
communication,
+ contain_service_on_interrupt=contain_service_on_interrupt,
stop_main_on_interrupt=stop_main_on_interrupt,
)
)
try:
await _await_task_uninterruptibly(cleanup, preserve_cancellation=False)
- except RuntimeError as exc:
- raise RuntimeError(
- "main task container containment could not be confirmed during cancellation"
- ) from exc
+ except BaseException as cleanup_error:
+ primary_error.add_note(
+ f"{containment_target} containment could not be confirmed during cancellation: "
+ f"{type(cleanup_error).__name__}: {cleanup_error}"
+ )
+ raise primary_error from cleanup_error
+ raise
+ except BaseException as exc:
+ await cleanup_preserving_primary(exc)
raise
- stdout = stdout_bytes.decode(errors="replace") if stdout_bytes else None
- stderr = stderr_bytes.decode(errors="replace") if stderr_bytes else None
- result = _redact_result(
- ExecResult(stdout=stdout, stderr=stderr, return_code=process.returncode or 0),
- secret_values,
- )
+ if callback_failure is not None:
+ # Kept outside the try/except above so a callback's deliberate
+ # CancelledError is not mistaken for cancellation of this caller.
+ await cleanup_preserving_primary(callback_failure)
+ raise callback_failure
+
if check and result.return_code != 0:
- raise RuntimeError(
+ detail = (
f"Docker compose command failed for environment {self.environment_name}. "
f"Command: {' '.join(full_command)}. Return code: {result.return_code}. "
f"Stdout: {result.stdout}. Stderr: {result.stderr}."
)
- return result
+ raise RuntimeError(
+ _redact(
+ detail,
+ secret_values,
+ replacement=redaction_marker,
+ include_short=True,
+ )
+ )
+ return _redact_result(
+ result,
+ secret_values,
+ replacement=redaction_marker,
+ include_short=True,
+ )
class SkillEvaluatorSecureDockerEnvironment(SkillEvaluatorDockerEnvironment):
@@ -418,13 +3344,15 @@ async def _exec_without_environment(
exec_command.extend(["-u", str(user)])
exec_command.append("main")
exec_command.extend(self._platform.exec_shell_args(command))
- result = await self._run_docker_compose_command(
+ return await self._run_docker_compose_command(
exec_command,
check=False,
timeout_sec=timeout_sec,
+ on_output=self._output_callback(),
+ additional_secret_values=secret_values,
stop_main_on_interrupt=True,
+ **({"exact_secret_values": secret_values} if secret_values else {}),
)
- return _redact_result(result, secret_values or set())
async def _remove_handoff(self, remote_path: str) -> None:
result = await self._run_docker_compose_command(
@@ -453,10 +3381,21 @@ async def exec(
)
merged = _host_handoff_environment(merged)
+ secret_values = set(_eligible_secret_values(merged.values()))
+ secret_values.update(_sensitive_environment_values(merged))
+ secret_values.update(_credential_uri_environment_values(merged))
+ if secret_values:
+ # Fail before writing or uploading a handoff if no structurally
+ # safe output marker can represent these values.
+ _collision_safe_redaction_marker(secret_values, include_short=True)
remote_path = f"/tmp/.skillevaluator-exec-env-{uuid.uuid4().hex}.sh"
+ handoff_scope = _SecureHandoffScope(
+ environment_names=frozenset(merged),
+ secret_values=frozenset(secret_values),
+ )
+ scope_token = _SECURE_HANDOFF_SCOPES.set((*_SECURE_HANDOFF_SCOPES.get(), handoff_scope))
primary_error: BaseException | None = None
try:
- secret_values = {value for value in merged.values() if value and len(value) >= _MIN_EXACT_SECRET_LENGTH}
await self._run_docker_compose_command(
[
"exec",
@@ -471,23 +3410,30 @@ async def exec(
remote_path,
],
check=True,
- stdin_bytes=_render_environment_script(merged).encode("utf-8"),
- redact_values=secret_values,
+ stdin_data=_render_environment_script(merged).encode("utf-8"),
+ additional_secret_values=secret_values,
+ exact_secret_values=secret_values,
)
if user is None:
await self._run_docker_compose_command(
["exec", "-u", "root", "main", "chmod", "600", remote_path],
check=True,
+ additional_secret_values=secret_values,
+ exact_secret_values=secret_values,
)
else:
await self._run_docker_compose_command(
["exec", "-u", "root", "main", "chown", "--", str(user), remote_path],
check=True,
+ additional_secret_values=secret_values,
+ exact_secret_values=secret_values,
)
await self._run_docker_compose_command(
["exec", "-u", "root", "main", "chmod", "600", remote_path],
check=True,
+ additional_secret_values=secret_values,
+ exact_secret_values=secret_values,
)
quoted_path = shlex.quote(remote_path)
wrapped = (
@@ -505,19 +3451,22 @@ async def exec(
primary_error = exc
raise
finally:
- cleanup = asyncio.create_task(self._remove_handoff(remote_path))
try:
- await _await_task_uninterruptibly(
- cleanup,
- preserve_cancellation=primary_error is None,
- )
- except Exception as cleanup_error:
- message = f"could not confirm removal of Docker environment handoff {remote_path}"
- if primary_error is not None:
- if hasattr(primary_error, "add_note"):
- primary_error.add_note(f"{message}: {cleanup_error}")
- else:
- raise RuntimeError(message) from cleanup_error
+ cleanup = asyncio.create_task(self._remove_handoff(remote_path))
+ try:
+ await _await_task_uninterruptibly(
+ cleanup,
+ preserve_cancellation=primary_error is None,
+ )
+ except Exception as cleanup_error:
+ message = f"could not confirm removal of Docker environment handoff {remote_path}"
+ if primary_error is not None:
+ if hasattr(primary_error, "add_note"):
+ primary_error.add_note(f"{message}: {cleanup_error}")
+ else:
+ raise RuntimeError(message) from cleanup_error
+ finally:
+ _SECURE_HANDOFF_SCOPES.reset(scope_token)
async def exec_with_sensitive_env(
self,
diff --git a/src/skillevaluator/tier3/harbor/stream_redaction.py b/src/skillevaluator/tier3/harbor/stream_redaction.py
new file mode 100644
index 00000000..11fd4521
--- /dev/null
+++ b/src/skillevaluator/tier3/harbor/stream_redaction.py
@@ -0,0 +1,558 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Boundary-safe exact-value redaction for streamed Harbor output."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections import deque
+from collections.abc import Iterable
+
+_REDACTION_LABEL = ""
+_REDACTION_SENTINEL_CANDIDATES = ("␟", "␞", "␝", "␜", "")
+_MAX_REDACTION_SCAN_CHUNK = 64 * 1024
+MAX_COMMAND_OUTPUT_BYTES = 16 * 1024 * 1024
+
+
+class CommandOutputLimitError(RuntimeError):
+ """Signal that an untrusted command exceeded its raw output budget."""
+
+
+class CommandOutputByteBudget:
+ """Enforce one combined raw stdout/stderr budget for a command."""
+
+ def __init__(self, limit_bytes: int | None = None) -> None:
+ self.limit_bytes = MAX_COMMAND_OUTPUT_BYTES if limit_bytes is None else limit_bytes
+ if self.limit_bytes <= 0:
+ raise ValueError("command output byte limit must be positive")
+ self.consumed_bytes = 0
+
+ def consume(self, chunk: bytes) -> None:
+ next_total = self.consumed_bytes + len(chunk)
+ if next_total > self.limit_bytes:
+ raise CommandOutputLimitError(f"Command output exceeded the {self.limit_bytes}-byte safety limit")
+ self.consumed_bytes = next_total
+
+
+def collision_safe_redaction_marker(secret_values: Iterable[str]) -> str:
+ """Build a readable marker that cannot contain or join into a secret."""
+ secrets = sorted({value for value in secret_values if value})
+ if not secrets:
+ return _REDACTION_LABEL
+
+ sentinel = next(
+ (
+ candidate
+ for candidate in _REDACTION_SENTINEL_CANDIDATES
+ if all(candidate not in secret for secret in secrets)
+ ),
+ None,
+ )
+ if sentinel is None:
+ used_characters = set().union(*map(set, secrets))
+ private_use_ranges = (
+ range(0xE000, 0xF900),
+ range(0xF0000, 0xFFFFE),
+ range(0x100000, 0x10FFFE),
+ )
+ for candidate_range in private_use_ranges:
+ sentinel = next(
+ (chr(codepoint) for codepoint in candidate_range if chr(codepoint) not in used_characters),
+ None,
+ )
+ if sentinel is not None:
+ break
+ if sentinel is None:
+ scalar_ranges = (range(1, 0xD800), range(0xE000, 0x110000))
+ for scalar_range in scalar_ranges:
+ sentinel = next(
+ (
+ chr(codepoint)
+ for codepoint in scalar_range
+ if chr(codepoint) not in used_characters and unicodedata.category(chr(codepoint))[0] in "LNPS"
+ ),
+ None,
+ )
+ if sentinel is not None:
+ break
+ if sentinel is None:
+ raise RuntimeError("Could not construct a collision-safe redaction marker")
+
+ minimum_secret_length = min(map(len, secrets))
+ if minimum_secret_length == 1:
+ return sentinel
+ chunk_length = minimum_secret_length - 1
+ label_chunks = [
+ _REDACTION_LABEL[index : index + chunk_length] for index in range(0, len(_REDACTION_LABEL), chunk_length)
+ ]
+ return sentinel + sentinel.join(label_chunks) + sentinel
+
+
+class _SecretTrieNode:
+ __slots__ = ("children", "depth", "failure", "max_terminal_length", "terminal_length")
+
+ def __init__(self) -> None:
+ self.children: dict[str, _SecretTrieNode] = {}
+ self.depth = 0
+ self.failure = self
+ self.max_terminal_length = 0
+ self.terminal_length = 0
+
+
+class StreamingSecretRedactor:
+ """Redact exact-value match unions with partition-linear work.
+
+ This is an Aho-Corasick matcher: each character advances through at most
+ one successful edge plus amortized failure edges, independent of secret
+ count and input partitioning. When the automaton is at its root, a C-level
+ character-class search skips spans that cannot begin any secret. Coverage
+ and output are committed in slices, keeping dense short-secret output both
+ memory-bounded and fast.
+ """
+
+ def __init__(self, secret_values: Iterable[str]) -> None:
+ secrets = sorted({value for value in secret_values if value})
+ self._root = _SecretTrieNode()
+ for secret in secrets:
+ node = self._root
+ for character in secret:
+ child = node.children.get(character)
+ if child is None:
+ child = _SecretTrieNode()
+ child.depth = node.depth + 1
+ node.children[character] = child
+ node = child
+ node.terminal_length = len(secret)
+
+ failure_queue = deque(self._root.children.values())
+ for child in failure_queue:
+ child.failure = self._root
+ child.max_terminal_length = child.terminal_length
+ while failure_queue:
+ node = failure_queue.popleft()
+ for character, child in node.children.items():
+ failure = node.failure
+ while failure is not self._root and character not in failure.children:
+ failure = failure.failure
+ child.failure = failure.children.get(character, self._root)
+ child.max_terminal_length = max(child.terminal_length, child.failure.max_terminal_length)
+ failure_queue.append(child)
+
+ self._has_secrets = bool(secrets)
+ self._max_secret_length = max(map(len, secrets), default=0)
+ self._single_character_re = (
+ re.compile("[" + "".join(re.escape(secret) for secret in secrets) + "]+")
+ if self._max_secret_length == 1
+ else None
+ )
+ self._state = self._root
+ self._first_character_re = (
+ re.compile("[" + "".join(re.escape(character) for character in self._root.children) + "]")
+ if self._root.children
+ else None
+ )
+ self._pending = ""
+ self._pending_start = 0
+ self._processed = 0
+ self._coverage: deque[tuple[int, int]] = deque()
+ self._redaction_open = False
+ self._replacement = collision_safe_redaction_marker(secrets)
+
+ def _add_coverage(self, start: int, end: int) -> None:
+ merged_start = start
+ while self._coverage and self._coverage[-1][1] >= merged_start:
+ previous_start, _previous_end = self._coverage.pop()
+ merged_start = min(merged_start, previous_start)
+ self._coverage.append((merged_start, end))
+
+ def _advance(self, character: str) -> None:
+ while True:
+ child = self._state.children.get(character)
+ if child is not None:
+ self._state = child
+ break
+ if self._state is self._root:
+ break
+ self._state = self._state.failure
+ self._processed += 1
+ if match_length := self._state.max_terminal_length:
+ self._add_coverage(self._processed - match_length, self._processed)
+
+ def _drain(self, *, final: bool) -> str:
+ # The automaton state is the longest current suffix that can still
+ # grow into a secret. Everything before that suffix is irrevocably
+ # safe; retaining a maximum-secret window would delay callbacks even
+ # after a mismatch returned to the root.
+ safe_end = self._processed if final else self._processed - self._state.depth
+ if safe_end <= self._pending_start:
+ return ""
+
+ emitted: list[str] = []
+ cursor = self._pending_start
+ while self._coverage and self._coverage[0][0] < safe_end:
+ start, end = self._coverage[0]
+ if start > cursor:
+ emitted.append(self._pending[cursor - self._pending_start : start - self._pending_start])
+ self._redaction_open = False
+ if not self._redaction_open:
+ emitted.append(self._replacement)
+ self._redaction_open = True
+ cursor = min(end, safe_end)
+ if end <= safe_end:
+ self._coverage.popleft()
+ else:
+ break
+ if cursor < safe_end:
+ emitted.append(self._pending[cursor - self._pending_start : safe_end - self._pending_start])
+ self._redaction_open = False
+ committed = safe_end - self._pending_start
+ self._pending = self._pending[committed:]
+ self._pending_start = safe_end
+ return "".join(emitted)
+
+ def _feed_chunk(self, text: str) -> str:
+ offset = 0
+ pending_parts = [self._pending]
+ while offset < len(text):
+ if self._state is self._root:
+ first_character_re = self._first_character_re
+ if first_character_re is None:
+ raise RuntimeError("exact redactor invariant violated")
+ match = first_character_re.search(text, offset)
+ end = len(text) if match is None else match.start()
+ if end > offset:
+ plain = text[offset:end]
+ pending_parts.append(plain)
+ self._processed += len(plain)
+ offset = end
+ if match is None:
+ break
+ character = text[offset]
+ pending_parts.append(character)
+ self._advance(character)
+ offset += 1
+ self._pending = "".join(pending_parts)
+ return self._drain(final=False)
+
+ def feed(self, text: str, *, final: bool = False) -> str:
+ """Return safe text while retaining one maximum-pattern window."""
+ if not self._has_secrets:
+ return text
+ if self._single_character_re is not None:
+ emitted: list[str] = []
+ cursor = 0
+ for match in self._single_character_re.finditer(text):
+ if match.start() > cursor:
+ emitted.append(text[cursor : match.start()])
+ self._redaction_open = False
+ if not self._redaction_open:
+ emitted.append(self._replacement)
+ self._redaction_open = True
+ cursor = match.end()
+ if cursor < len(text):
+ emitted.append(text[cursor:])
+ self._redaction_open = False
+ return "".join(emitted)
+
+ emitted = "".join(
+ self._feed_chunk(text[offset : offset + _MAX_REDACTION_SCAN_CHUNK])
+ for offset in range(0, len(text), _MAX_REDACTION_SCAN_CHUNK)
+ )
+ if final:
+ emitted += self._drain(final=True)
+ return emitted
+
+ def finish(self) -> str:
+ """Flush the suffix once later input cannot complete a secret."""
+ return self.feed("", final=True)
+
+
+class StreamingLogRedactor:
+ """Redact known key shapes and exact values across arbitrary chunks.
+
+ Each family runs once on either side of the other. The post-known pass
+ catches a token boundary created by an exact-value marker; the final exact
+ pass catches a configured value synthesized by a known-shape replacement.
+ Collision-safe exact markers cannot themselves create either family.
+ """
+
+ def __init__(self, secret_values: Iterable[str]) -> None:
+ self._known_redactor = _StreamingKnownPatternRedactor()
+ self._exact_redactor = StreamingSecretRedactor(secret_values)
+ self._post_known_redactor = _StreamingKnownPatternRedactor()
+ self._post_exact_redactor = StreamingSecretRedactor(secret_values)
+
+ def feed(self, text: str) -> str:
+ known = self._known_redactor.feed(text)
+ exact = self._exact_redactor.feed(known)
+ post_known = self._post_known_redactor.feed(exact)
+ return self._post_exact_redactor.feed(post_known)
+
+ def finish(self) -> str:
+ known = self._known_redactor.finish()
+ exact = self._exact_redactor.feed(known) + self._exact_redactor.finish()
+ post_known = self._post_known_redactor.feed(exact) + self._post_known_redactor.finish()
+ return self._post_exact_redactor.feed(post_known) + self._post_exact_redactor.finish()
+
+
+_KNOWN_PREFIX_RE = re.compile(r"sk-|nvapi-|crsr_|sha256~|eyJ")
+_KNOWN_PREFIXES = ("sk-", "nvapi-", "crsr_", "sha256~", "eyJ")
+_MAX_KNOWN_PREFIX_LENGTH = len("sha256~")
+_ASCII_KEY_BODY_RE = re.compile(r"[^A-Za-z0-9_-]")
+_ASCII_ALNUM_RE = re.compile(r"[^A-Za-z0-9]")
+_ASCII_HEX_RE = re.compile(r"[^a-f0-9]")
+_OPENSHIFT_BODY_RE = re.compile(r"[^A-Za-z0-9._~-]")
+_JWT_SEGMENT_RE = re.compile(r"[^A-Za-z0-9_-]")
+_JWT_TOKEN_RE = re.compile(r"[^A-Za-z0-9_.-]")
+_GLUED_KEY_BUFFER_LIMIT = 256
+_JWT_CANDIDATE_BUFFER_LIMIT = 256
+_KNOWN_REPLACEMENTS = {
+ "sk-": "sk-",
+ "nvapi-": "nvapi-",
+ "crsr_": "crsr_",
+ "sha256~": "sha256~",
+ "eyJ": "jwt-",
+}
+
+
+def _is_ascii_alnum(character: str) -> bool:
+ return character.isascii() and character.isalnum()
+
+
+def _is_ascii_key_body(character: str) -> bool:
+ return _is_ascii_alnum(character) or character in "_-"
+
+
+def _is_openshift_body(character: str) -> bool:
+ return _is_ascii_alnum(character) or character in "._~-"
+
+
+def _partial_known_prefix_length(text: str) -> int:
+ maximum = min(_MAX_KNOWN_PREFIX_LENGTH - 1, len(text))
+ return next(
+ (
+ length
+ for length in range(maximum, 0, -1)
+ if any(prefix.startswith(text[-length:]) for prefix in _KNOWN_PREFIXES)
+ ),
+ 0,
+ )
+
+
+class _StreamingKnownPatternRedactor:
+ """Recognize known secret shapes without buffering ordinary output.
+
+ Plain chunks are scanned by the regular-expression engine and retain only
+ the longest possible partial prefix. Once a prefix is found, the small
+ deterministic recognizer below either rejects it or proves it secret. A
+ proven token emits its marker immediately and discards the remainder with
+ a compiled character-class search, so attacker-sized tokens do not grow a
+ Python list or monopolize the event loop.
+
+ The glued ``sk-``/``nvapi-`` alternative can otherwise remain ambiguous
+ forever while waiting for its lower/upper/digit mix. After a bounded
+ prefix it is conservatively redacted. That favors non-disclosure for an
+ adversarial token while preserving the canonical behavior for normal
+ candidates and all boundary-prefixed keys.
+ """
+
+ def __init__(self) -> None:
+ self._plain_tail = ""
+ self._candidate_kind: str | None = None
+ self._candidate_prefix = ""
+ self._candidate: list[str] = []
+ self._candidate_count = 0
+ self._candidate_has_lower = False
+ self._candidate_has_upper = False
+ self._candidate_has_digit = False
+ self._jwt_stage = 0
+ self._discard_re: re.Pattern[str] | None = None
+ self._previous_raw_character = ""
+
+ def _reset_candidate(self) -> None:
+ self._candidate_kind = None
+ self._candidate_prefix = ""
+ self._candidate = []
+ self._candidate_count = 0
+ self._candidate_has_lower = False
+ self._candidate_has_upper = False
+ self._candidate_has_digit = False
+ self._jwt_stage = 0
+
+ def _start_candidate(self, prefix: str) -> None:
+ self._candidate_prefix = prefix
+ self._candidate = [prefix]
+ boundary_character = self._previous_raw_character
+ if prefix in {"sk-", "nvapi-"}:
+ self._candidate_kind = (
+ "boundary-key" if not boundary_character or not _is_ascii_key_body(boundary_character) else "glued-key"
+ )
+ elif prefix == "crsr_":
+ self._candidate_kind = (
+ "crsr" if not boundary_character or not _is_ascii_key_body(boundary_character) else "invalid"
+ )
+ elif prefix == "sha256~":
+ self._candidate_kind = (
+ "openshift" if not boundary_character or not _is_ascii_key_body(boundary_character) else "invalid"
+ )
+ else:
+ self._candidate_kind = (
+ "jwt"
+ if not boundary_character or not (boundary_character.isalnum() or boundary_character == "_")
+ else "invalid"
+ )
+
+ def _prove_candidate(self, emitted: list[str], discard_re: re.Pattern[str]) -> None:
+ replacement = _KNOWN_REPLACEMENTS[self._candidate_prefix]
+ emitted.append(replacement)
+ # Later canonical redactors see the replacement, not the raw token.
+ # Retaining that boundary also catches an adjacent lower-priority shape
+ # such as ``crsr_sha256~...``.
+ self._previous_raw_character = replacement[-1]
+ self._discard_re = discard_re
+ self._reset_candidate()
+
+ def _reject_candidate(self, pending: str, offset: int, emitted: list[str]) -> str:
+ raw_candidate = "".join(self._candidate)
+ # Emitting one character guarantees progress while allowing every
+ # nested prefix in the bounded remainder to be recognized normally.
+ emitted.append(raw_candidate[0])
+ self._previous_raw_character = raw_candidate[0]
+ self._reset_candidate()
+ return raw_candidate[1:] + pending[offset:]
+
+ def _consume_candidate(self, pending: str, emitted: list[str], *, final: bool) -> str:
+ offset = 0
+ while offset < len(pending):
+ character = pending[offset]
+ kind = self._candidate_kind
+ if kind == "invalid":
+ return self._reject_candidate(pending, offset, emitted)
+
+ if kind == "boundary-key":
+ if not _is_ascii_key_body(character):
+ return self._reject_candidate(pending, offset, emitted)
+ self._candidate.append(character)
+ self._candidate_count += 1
+ offset += 1
+ if self._candidate_count == 8:
+ self._prove_candidate(emitted, _ASCII_KEY_BODY_RE)
+ return pending[offset:]
+ continue
+
+ if kind == "glued-key":
+ if not _is_ascii_alnum(character):
+ return self._reject_candidate(pending, offset, emitted)
+ self._candidate.append(character)
+ self._candidate_count += 1
+ self._candidate_has_lower |= character.islower()
+ self._candidate_has_upper |= character.isupper()
+ self._candidate_has_digit |= character.isdigit()
+ offset += 1
+ if self._candidate_count >= 20 and (
+ self._candidate_has_lower and self._candidate_has_upper and self._candidate_has_digit
+ ):
+ self._prove_candidate(emitted, _ASCII_ALNUM_RE)
+ return pending[offset:]
+ if self._candidate_count == _GLUED_KEY_BUFFER_LIMIT:
+ self._prove_candidate(emitted, _ASCII_ALNUM_RE)
+ return pending[offset:]
+ continue
+
+ if kind == "crsr":
+ if character not in "abcdef0123456789":
+ return self._reject_candidate(pending, offset, emitted)
+ self._candidate.append(character)
+ self._candidate_count += 1
+ offset += 1
+ if self._candidate_count == 16:
+ self._prove_candidate(emitted, _ASCII_HEX_RE)
+ return pending[offset:]
+ continue
+
+ if kind == "openshift":
+ if not _is_openshift_body(character):
+ return self._reject_candidate(pending, offset, emitted)
+ self._candidate.append(character)
+ offset += 1
+ self._prove_candidate(emitted, _OPENSHIFT_BODY_RE)
+ return pending[offset:]
+
+ if kind != "jwt":
+ raise RuntimeError("known-pattern redactor invariant violated")
+ if _is_ascii_key_body(character):
+ self._candidate.append(character)
+ self._candidate_count += 1
+ offset += 1
+ if self._jwt_stage == 2 and self._candidate_count == 20:
+ # Waiting for a Unicode word/non-word boundary would make
+ # the final segment unbounded. At this point the complete
+ # JWT shape is present, so redact conservatively and stream.
+ self._prove_candidate(emitted, _JWT_SEGMENT_RE)
+ return pending[offset:]
+ if self._jwt_stage < 2 and self._candidate_count == _JWT_CANDIDATE_BUFFER_LIMIT:
+ # A stage-zero/stage-one JWT candidate can otherwise stay
+ # ambiguous until attacker-controlled EOF. Conservatively
+ # redact after a bounded prefix, then discard the rest of
+ # token-like continuation with the compiled scanner.
+ self._prove_candidate(emitted, _JWT_TOKEN_RE)
+ return pending[offset:]
+ continue
+ if character != "." or self._jwt_stage >= 2 or self._candidate_count < 20:
+ return self._reject_candidate(pending, offset, emitted)
+ self._candidate.append(character)
+ self._jwt_stage += 1
+ self._candidate_count = 0
+ offset += 1
+
+ if final:
+ return self._reject_candidate(pending, offset, emitted)
+ return ""
+
+ def feed(self, text: str) -> str:
+ return self._feed(text, final=False)
+
+ def _feed(self, text: str, *, final: bool) -> str:
+ emitted: list[str] = []
+ pending = self._plain_tail + text
+ self._plain_tail = ""
+ while pending or (final and self._candidate_kind is not None):
+ if self._discard_re is not None:
+ end = self._discard_re.search(pending)
+ if end is None:
+ pending = ""
+ continue
+ pending = pending[end.start() :]
+ self._discard_re = None
+ continue
+
+ if self._candidate_kind is not None:
+ pending = self._consume_candidate(pending, emitted, final=final)
+ continue
+
+ match = _KNOWN_PREFIX_RE.search(pending)
+ if match is not None:
+ plain = pending[: match.start()]
+ if plain:
+ emitted.append(plain)
+ self._previous_raw_character = plain[-1]
+ self._start_candidate(match.group())
+ pending = pending[match.end() :]
+ continue
+
+ retained = 0 if final else _partial_known_prefix_length(pending)
+ if retained:
+ plain = pending[:-retained]
+ self._plain_tail = pending[-retained:]
+ else:
+ plain = pending
+ if plain:
+ emitted.append(plain)
+ self._previous_raw_character = plain[-1]
+ pending = ""
+ return "".join(emitted)
+
+ def finish(self) -> str:
+ return self._feed("", final=True)
diff --git a/src/skillevaluator/tier3/harbor/templates/custom_grader_runner.py b/src/skillevaluator/tier3/harbor/templates/custom_grader_runner.py
index c5fb8005..1c6f56bd 100644
--- a/src/skillevaluator/tier3/harbor/templates/custom_grader_runner.py
+++ b/src/skillevaluator/tier3/harbor/templates/custom_grader_runner.py
@@ -8,7 +8,9 @@
import argparse
import json
+import math
import os
+import re
import subprocess
import sys
from pathlib import Path
@@ -39,14 +41,19 @@ def _env_path(name, default):
"accuracy",
"goal_accuracy",
"behavior_check",
+ "custom_details",
+ "custom_metrics",
"details",
"entry_id",
+ "error",
"evaluation_errors",
"evaluation_status",
"has_skill",
"metric_set",
+ "metric_set_version",
"metrics",
"overall",
+ "trajectory_detail",
"trajectory_source",
}
DEFAULT_METRICS = {
@@ -57,6 +64,86 @@ def _env_path(name, default):
"goal_accuracy",
"behavior_check",
}
+MAX_CUSTOM_METRICS = 128
+MAX_CUSTOM_METRIC_NAME_BYTES = 256
+_SAFE_SENSITIVE_METRIC_PREFIXES = {"auth", "secret", "token"}
+_SAFE_SENSITIVE_METRIC_SUFFIXES = {
+ "accuracy",
+ "compliance",
+ "count",
+ "coverage",
+ "efficiency",
+ "handling",
+ "leakage",
+ "precision",
+ "quality",
+ "rate",
+ "ratio",
+ "recall",
+ "safety",
+ "score",
+ "usage",
+}
+_SENSITIVE_KEY_PARTS = {
+ "auth",
+ "authorization",
+ "bearer",
+ "credential",
+ "credentials",
+ "key",
+ "password",
+ "private",
+ "secret",
+ "token",
+}
+_PLURAL_SENSITIVE_KEY_PARTS = {
+ "auths",
+ "authorizations",
+ "bearers",
+ "credentials",
+ "passwords",
+ "secrets",
+ "tokens",
+}
+_TOKEN_COUNT_KEYS = {
+ "cached_tokens",
+ "completion_tokens",
+ "expected_max_tokens",
+ "frontmatter_tokens",
+ "input_tokens",
+ "instructions_tokens",
+ "n_input_tokens",
+ "n_cache_tokens",
+ "n_output_tokens",
+ "output_tokens",
+ "prompt_tokens",
+ "reasoning_output_tokens",
+ "last_token_usage",
+ "max_completion_tokens",
+ "max_output_tokens",
+ "max_tokens",
+ "recommended_max_tokens",
+ "token_count",
+ "tokens",
+ "total_cached_tokens",
+ "total_completion_tokens",
+ "total_prompt_tokens",
+ "total_tokens",
+}
+_EMBEDDED_CREDENTIAL_NAME_RE = re.compile(
+ r"(?:sk-|nvapi-)[a-zA-Z0-9_-]{8,}"
+ r"|crsr_[a-f0-9]{16,}"
+ r"|(?:AKIA|ASIA)[A-Z0-9]{16}"
+ r"|eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"
+ r"|sha256~[A-Za-z0-9._~-]+"
+ r"|(?i:gh[pour]_[a-z0-9]{36})"
+ r"|(?i:ghs_[a-z0-9.\-_]{36,})"
+ r"|(?i:github_pat_[a-z0-9_]{20,})"
+ r"|(?i:xox[baprs]-[a-z0-9-]{10,})"
+ r"|AIza[A-Za-z0-9_-]{20,}"
+ r"|(?i:glpat-[a-z0-9_-]{20,})"
+)
+_CREDENTIAL_URI_NAME_RE = re.compile(r"(?i)[a-z][a-z0-9+.-]{0,31}://[^\s/?#]*@")
def _load_json(path: Path) -> dict[str, Any]:
@@ -70,21 +157,65 @@ def _load_json(path: Path) -> dict[str, Any]:
def _numeric(value: Any) -> float | None:
- if isinstance(value, int | float) and not isinstance(value, bool):
- return float(value)
- return None
+ if not isinstance(value, int | float) or isinstance(value, bool):
+ return None
+ try:
+ score = float(value)
+ except (OverflowError, ValueError):
+ return None
+ return score if math.isfinite(score) and 0.0 <= score <= 1.0 else None
+
+
+def _normalized_metric_name_parts(name: str) -> tuple[str, ...]:
+ camel_split = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
+ normalized = re.sub(r"[^a-zA-Z0-9]+", "_", camel_split).strip("_").lower()
+ return tuple(part for part in normalized.split("_") if part)
+
+
+def _metric_name_is_publishable(name: object) -> bool:
+ text = str(name)
+ try:
+ encoded = text.encode("utf-8")
+ except UnicodeError:
+ return False
+ if not text or text != text.strip() or not text.isprintable() or len(encoded) > MAX_CUSTOM_METRIC_NAME_BYTES:
+ return False
+ return not _metric_name_contains_sensitive_data(text)
+
+
+def _metric_name_contains_sensitive_data(text: str) -> bool:
+ if _EMBEDDED_CREDENTIAL_NAME_RE.search(text) or _CREDENTIAL_URI_NAME_RE.search(text):
+ return True
+ parts = _normalized_metric_name_parts(text)
+ if "_".join(parts) in _TOKEN_COUNT_KEYS:
+ return False
+ part_set = set(parts)
+ compact = "".join(parts)
+ sensitive = bool(part_set & (_SENSITIVE_KEY_PARTS | _PLURAL_SENSITIVE_KEY_PARTS))
+ sensitive = sensitive or "apikey" in compact or "accesskey" in compact
+ sensitive = sensitive or "privatekey" in compact or "sessiontoken" in compact
+ explicitly_safe = (
+ len(parts) == 2 and parts[0] in _SAFE_SENSITIVE_METRIC_PREFIXES and parts[1] in _SAFE_SENSITIVE_METRIC_SUFFIXES
+ )
+ return sensitive and not explicitly_safe
+
+
+def _metric_name_shape_is_valid(name: object) -> bool:
+ text = str(name)
+ try:
+ encoded = text.encode("utf-8")
+ except UnicodeError:
+ return False
+ return bool(text) and text == text.strip() and text.isprintable() and len(encoded) <= MAX_CUSTOM_METRIC_NAME_BYTES
def _score_from_reward(reward: dict[str, Any]) -> float | None:
- score = _numeric(reward.get("overall"))
- if score is not None:
- return max(0.0, min(1.0, score))
- return None
+ return _numeric(reward.get("overall"))
def _score_from_text(text: str) -> float | None:
try:
- return max(0.0, min(1.0, float(text.strip())))
+ return _numeric(float(text.strip()))
except ValueError:
return None
@@ -103,31 +234,92 @@ def _extract_custom_metrics(reward: dict[str, Any]) -> dict[str, float]:
if key in reward:
raise RuntimeError(f"Custom grader cannot overwrite reserved SkillEvaluator metric '{key}'")
explicit = reward.get("custom_metrics")
+ if "custom_metrics" in reward and not isinstance(explicit, dict):
+ raise RuntimeError("Custom metrics container must be a JSON object")
+
+ sources: list[tuple[dict[Any, Any], bool]] = []
if isinstance(explicit, dict):
- source = explicit
- else:
- source = {key: value for key, value in reward.items() if key not in RESERVED and not str(key).startswith("_")}
-
- for key, value in source.items():
- if key in RESERVED:
- raise RuntimeError(f"Custom metric '{key}' collides with reserved SkillEvaluator metric names")
- if isinstance(value, dict):
- value = value.get("score")
- score = _numeric(value)
- if score is not None:
- custom[str(key)] = max(0.0, min(1.0, score))
+ sources.append((explicit, True))
+ metrics = reward.get("metrics")
+ if isinstance(metrics, dict):
+ sources.append((metrics, False))
+ sources.append(
+ (
+ {key: value for key, value in reward.items() if str(key) not in RESERVED and not str(key).startswith("_")},
+ False,
+ )
+ )
+
+ for source, reject_reserved in sources:
+ for key, value in source.items():
+ name = str(key)
+ if name in RESERVED:
+ if reject_reserved:
+ raise RuntimeError(f"Custom metric '{key}' collides with reserved SkillEvaluator metric names")
+ continue
+ if isinstance(value, dict):
+ value = value.get("score")
+ score = _numeric(value)
+ if score is None:
+ continue
+ if _metric_name_contains_sensitive_data(name):
+ continue
+ if not _metric_name_shape_is_valid(name):
+ raise RuntimeError("Custom metric name exceeds the bounded publication contract")
+ if name not in custom and len(custom) >= MAX_CUSTOM_METRICS:
+ raise RuntimeError("Custom metric count exceeds the per reward publication limit")
+ custom[name] = max(0.0, min(1.0, score))
return custom
+def _sanitized_custom_reward(reward: dict[str, Any], custom_metrics: dict[str, float]) -> dict[str, Any]:
+ """Keep custom evidence only for validated metric names."""
+ sanitized = dict(reward)
+ explicit = reward.get("custom_metrics")
+ if isinstance(explicit, dict):
+ sanitized["custom_metrics"] = {
+ str(raw_name): custom_metrics[str(raw_name)] for raw_name in explicit if str(raw_name) in custom_metrics
+ }
+ metrics = reward.get("metrics")
+ if isinstance(metrics, dict):
+ sanitized["metrics"] = {
+ str(raw_name): value for raw_name, value in metrics.items() if str(raw_name) in custom_metrics
+ }
+ for raw_name, value in list(reward.items()):
+ name = str(raw_name)
+ if name in RESERVED or name.startswith("_"):
+ continue
+ if not _metric_name_is_publishable(name):
+ sanitized.pop(raw_name, None)
+ continue
+ candidate = value.get("score") if isinstance(value, dict) else value
+ if _numeric(candidate) is None:
+ continue
+ if name not in custom_metrics:
+ sanitized.pop(raw_name, None)
+ for field in ("details", "custom_details"):
+ details = reward.get(field)
+ if not isinstance(details, dict):
+ continue
+ safe_details = {
+ str(raw_name): detail for raw_name, detail in details.items() if str(raw_name) in custom_metrics
+ }
+ if safe_details:
+ sanitized[field] = safe_details
+ else:
+ sanitized.pop(field, None)
+ return sanitized
+
+
def _numeric_reward_payload(reward: dict[str, Any], *, overall: float | None = None) -> dict[str, float]:
payload: dict[str, float] = {}
for key, value in reward.items():
- if isinstance(value, bool):
- continue
- if isinstance(value, int | float):
- payload[str(key)] = float(value)
- if overall is not None:
- payload["overall"] = float(overall)
+ numeric = _numeric(value)
+ if numeric is not None:
+ payload[str(key)] = numeric
+ numeric_overall = _numeric(overall)
+ if numeric_overall is not None:
+ payload["overall"] = numeric_overall
return payload
@@ -160,12 +352,14 @@ def _run_default_plus_custom() -> None:
_run_grader()
custom_reward = _load_json(REWARD_JSON)
- CUSTOM_REWARD_JSON.write_text(json.dumps(custom_reward, indent=2), encoding="utf-8")
+ custom_metrics = _extract_custom_metrics(custom_reward)
+ safe_custom_reward = _sanitized_custom_reward(custom_reward, custom_metrics)
+ CUSTOM_REWARD_JSON.write_text(json.dumps(safe_custom_reward, indent=2), encoding="utf-8")
- skill_evaluator_reward["custom_metrics"] = _extract_custom_metrics(custom_reward)
+ skill_evaluator_reward["custom_metrics"] = custom_metrics
if skill_evaluator_overall is not None:
skill_evaluator_reward["overall"] = skill_evaluator_overall
- custom_details = custom_reward.get("details")
+ custom_details = safe_custom_reward.get("details")
if isinstance(custom_details, dict):
skill_evaluator_reward["custom_details"] = custom_details
SKILL_EVALUATOR_REWARD_JSON.write_text(json.dumps(skill_evaluator_reward, indent=2), encoding="utf-8")
@@ -185,13 +379,13 @@ def _run_custom_only() -> None:
if score is None:
score = _score_from_txt()
if score is None:
- raise RuntimeError("custom_only requires numeric `overall` in reward.json or numeric reward.txt")
- if not 0.0 <= score <= 1.0:
- raise RuntimeError("custom_only overall score must be between 0.0 and 1.0")
+ raise RuntimeError("custom_only requires numeric `overall` between 0.0 and 1.0 in reward.json or reward.txt")
reward["overall"] = score
- CUSTOM_REWARD_JSON.write_text(json.dumps(reward, indent=2), encoding="utf-8")
- harbor_reward = _numeric_reward_payload(reward, overall=score)
- harbor_reward.update(_extract_custom_metrics(reward))
+ custom_metrics = _extract_custom_metrics(reward)
+ safe_reward = _sanitized_custom_reward(reward, custom_metrics)
+ CUSTOM_REWARD_JSON.write_text(json.dumps(safe_reward, indent=2), encoding="utf-8")
+ harbor_reward = _numeric_reward_payload(safe_reward, overall=score)
+ harbor_reward.update(custom_metrics)
REWARD_JSON.write_text(json.dumps(harbor_reward, indent=2), encoding="utf-8")
REWARD_TXT.write_text(str(score), encoding="utf-8")
diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.py
index 54ff6a7b..e0078e1f 100644
--- a/src/skillevaluator/tier3/harbor/templates/eval.py
+++ b/src/skillevaluator/tier3/harbor/templates/eval.py
@@ -3816,7 +3816,7 @@ def main():
if judge_errors:
result["evaluation_status"] = "failed"
result["evaluation_errors"] = judge_errors
- # Harbor 0.13.2 still parses reward.json when the verifier exits nonzero.
+ # Harbor 0.22 still parses reward.json when the verifier exits nonzero.
# Keep this artifact deliberately incomplete so the collector cannot
# score it even if the richer diagnostic sidecar is unavailable.
write_reward_outputs(result, 0.0)
diff --git a/src/skillevaluator/tier3/results_location.py b/src/skillevaluator/tier3/results_location.py
index 09c30fb5..a131cb08 100644
--- a/src/skillevaluator/tier3/results_location.py
+++ b/src/skillevaluator/tier3/results_location.py
@@ -530,9 +530,34 @@ def _legacy_pass_at_k_is_complete(
):
return False
extra_case_names = set(extra_cases)
+ extra_case_count = payload.get("extra_case_count", len(extra_cases))
+ extra_cases_truncated = payload.get("extra_cases_truncated", False)
+ if (
+ not _is_nonnegative_int(extra_case_count)
+ or not isinstance(extra_cases_truncated, bool)
+ or len(extra_cases) > extra_case_count
+ or extra_cases_truncated != (len(extra_cases) < extra_case_count)
+ ):
+ return False
cases = payload.get("cases")
if not isinstance(cases, dict) or not cases:
return False
+ case_details_total = payload.get("case_details_total", len(cases))
+ case_details_shown = payload.get("case_details_shown", len(cases))
+ case_details_truncated = payload.get("case_details_truncated", False)
+ case_details_limit = payload.get("case_details_limit", len(cases))
+ if (
+ not _is_nonnegative_int(case_details_total)
+ or not _is_nonnegative_int(case_details_shown)
+ or not _is_nonnegative_int(case_details_limit)
+ or not isinstance(case_details_truncated, bool)
+ or case_details_shown != len(cases)
+ or case_details_shown > case_details_limit
+ or case_details_shown > case_details_total
+ or case_details_truncated != (case_details_shown < case_details_total)
+ or case_details_total != total_cases + extra_case_count
+ ):
+ return False
total_attempt_rows = 0
expected_attempt_rows = 0
observed_passed_cases = 0
@@ -544,13 +569,22 @@ def _legacy_pass_at_k_is_complete(
attempts_skipped = case.get("attempts_skipped")
attempts_missing = case.get("attempts_missing")
attempts = case.get("attempts")
+ attempt_details_total = case.get("attempt_details_total", case_attempts_used)
+ attempt_details_shown = case.get("attempt_details_shown", len(attempts) if isinstance(attempts, list) else -1)
+ attempt_details_truncated = case.get("attempt_details_truncated", False)
if (
not isinstance(case.get("passed"), bool)
or not _is_nonnegative_int(case_attempts_used)
or not _is_nonnegative_int(attempts_skipped)
or not _is_nonnegative_int(attempts_missing)
or not isinstance(attempts, list)
- or len(attempts) != case_attempts_used
+ or not _is_nonnegative_int(attempt_details_total)
+ or not _is_nonnegative_int(attempt_details_shown)
+ or not isinstance(attempt_details_truncated, bool)
+ or attempt_details_total != case_attempts_used
+ or attempt_details_shown != len(attempts)
+ or attempt_details_shown > attempt_details_total
+ or attempt_details_truncated != (attempt_details_shown < attempt_details_total)
):
return False
first_pass_attempt = case.get("first_pass_attempt")
@@ -558,7 +592,7 @@ def _legacy_pass_at_k_is_complete(
not isinstance(first_pass_attempt, int)
or isinstance(first_pass_attempt, bool)
or first_pass_attempt < 1
- or first_pass_attempt > len(attempts)
+ or first_pass_attempt > case_attempts_used
):
return False
for ordinal, attempt in enumerate(attempts, start=1):
@@ -578,30 +612,38 @@ def _legacy_pass_at_k_is_complete(
attempt_scores = [attempt["score"] for attempt in attempts]
expected_best_score = max(attempt_scores) if attempt_scores else None
best_score = case.get("best_score")
- if (
- case["passed"] != bool(passing_ordinals)
- or first_pass_attempt != expected_first_pass
- or (
- (expected_best_score is None and best_score is not None)
- or (
- expected_best_score is not None
- and (
- not _is_finite_number(best_score)
- or not math.isclose(best_score, expected_best_score, abs_tol=1e-9)
- )
+ if not attempt_details_truncated:
+ best_score_disagrees = (expected_best_score is None and best_score is not None) or (
+ expected_best_score is not None
+ and (
+ not _is_finite_number(best_score) or not math.isclose(best_score, expected_best_score, abs_tol=1e-9)
)
)
- ):
- return False
- total_attempt_rows += len(attempts)
- if case_name not in extra_case_names:
+ if (
+ case["passed"] != bool(passing_ordinals)
+ or first_pass_attempt != expected_first_pass
+ or best_score_disagrees
+ ):
+ return False
+ total_attempt_rows += case_attempts_used
+ case_is_extra = case.get("extra_case") is True
+ if not case_is_extra:
observed_expected_cases += 1
observed_passed_cases += int(case["passed"])
- expected_attempt_rows += len(attempts)
+ expected_attempt_rows += case_attempts_used
if not allow_coverage_failure and case_attempts_used + attempts_skipped + attempts_missing != k:
return False
- elif case.get("extra_case") is not True:
+ elif not (extra_cases_truncated or case_details_truncated or case_name in extra_case_names):
return False
+ if case_details_truncated:
+ return (
+ observed_expected_cases <= total_cases
+ and observed_passed_cases <= passed_cases
+ and expected_attempt_rows <= attempts_used
+ and total_attempt_rows <= num_trials
+ and (expected_scored_attempts is None or attempts_used == expected_scored_attempts)
+ and (not require_scored_attempt or attempts_used > 0)
+ )
return (
extra_case_names.issubset(cases)
and observed_expected_cases == total_cases
diff --git a/src/skillevaluator/tier3_environments.py b/src/skillevaluator/tier3_environments.py
index d488f271..dc77bf9c 100644
--- a/src/skillevaluator/tier3_environments.py
+++ b/src/skillevaluator/tier3_environments.py
@@ -16,7 +16,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -25,13 +28,156 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
# Not a Harbor-native backend: SkillEvaluator's host execution mode, run
# under an OS sandbox (bubblewrap on Linux, Seatbelt on macOS). Dispatched
- # via --environment-import-path, not Harbor's --env.
+ # by passing its custom import path through Harbor's unified --env flag.
"local",
)
HARBOR_ENV_MODES = frozenset(HARBOR_ENVIRONMENTS)
#: env modes that Harbor accepts natively via ``--env`` (everything except ``local``).
HARBOR_NATIVE_ENV_MODES = frozenset(m for m in HARBOR_ENVIRONMENTS if m != "local")
+# Exact Harbor 0.22 ``Provides-Extra`` names. ``ack`` reuses the Kubernetes
+# dependencies supplied by ``gke``. Of the four ``None`` entries, Docker needs
+# no additional Python extra and the other three are system-CLI backends.
+HARBOR_ENVIRONMENT_EXTRAS: dict[str, str | None] = {
+ "docker": None,
+ "daytona": "daytona",
+ "e2b": "e2b",
+ "modal": "modal",
+ "runloop": "runloop",
+ "langsmith": "langsmith",
+ "ec2": "ec2",
+ "gke": "gke",
+ "ack": "gke",
+ "openshift": None,
+ "novita": "novita",
+ "apple-container": None,
+ "singularity": None,
+ "islo": "islo",
+ "tensorlake": "tensorlake",
+ "cwsandbox": "cwsandbox",
+ "wandb": "wandb",
+ "use-computer": "use-computer",
+ "cua-cloud": "cua",
+ "blaxel": "blaxel",
+ "opensandbox": "opensandbox",
+ "beam": "beam",
+ "skypilot": "skypilot",
+ "hf-sandbox": "hf-sandbox",
+ "hyperbrowser": "hyperbrowser",
+ "vercel": "vercel",
+}
+
+# Constructor kwargs consumed by Harbor 0.22.0. Keep this static so importing
+# the base SkillEvaluator CLI never imports Harbor or optional provider SDKs.
+# The packaging parity test AST-reads the pinned Harbor sources and catches
+# additions, removals, and provider kwargs that are consumed through **kwargs.
+_HARBOR_V022_BASE_ENVIRONMENT_KWARGS = frozenset(
+ {
+ "cpu_enforcement_policy",
+ "environment_dir",
+ "environment_name",
+ "extra_docker_compose",
+ "logger",
+ "memory_enforcement_policy",
+ "mounts",
+ "network_policy",
+ "override_cpus",
+ "override_gpus",
+ "override_memory_mb",
+ "override_storage_mb",
+ "override_tpu",
+ "persistent_env",
+ "phase_network_policies",
+ "session_id",
+ "suppress_override_warnings",
+ "task_env_config",
+ "trial_paths",
+ }
+)
+
+_HARBOR_V022_BACKEND_ENVIRONMENT_KWARGS = {
+ "docker": "keep_containers",
+ "daytona": (
+ "assume_global_snapshot auto_delete_interval_mins auto_labels auto_snapshot auto_stop_interval_mins "
+ "connection_pool_maxsize dind_image dind_snapshot expose_sandbox_id labels network_block_all secrets "
+ "snapshot_template_name"
+ ),
+ "e2b": "",
+ "modal": (
+ "app_name auto_labels dind_image keepalive labels modal_sandbox_v2 modal_vm_runtime region registry_secret "
+ "sandbox_idle_timeout_secs sandbox_timeout_secs secrets volumes"
+ ),
+ "runloop": "",
+ "langsmith": (
+ "api_key create_snapshot delete_after_stop_seconds delete_snapshot idle_ttl_seconds langsmith_api_key "
+ "langsmith_endpoint poll_interval_seconds registry_id request_timeout_seconds sandbox_api_url snapshot_name "
+ "startup_timeout_seconds ttl_seconds workdir"
+ ),
+ "ec2": (
+ "ami_id aws_profile bootstrap_docker compose_up_timeout_sec docker_ready_timeout_sec iam_instance_profile "
+ "instance_id instance_ready_timeout_sec instance_type key_name launch_mode region root_device_name "
+ "root_volume_size_gb root_volume_type security_group_ids ssh_connect_timeout_sec ssh_key_path "
+ "ssh_known_hosts_path ssh_port ssh_user strict_host_key_checking subnet_id tags use_public_ip"
+ ),
+ "gke": (
+ "cloud_build_disk_size_gb cloud_build_machine_type cluster_name dind_image memory_limit_multiplier namespace "
+ "project_id region registry_location registry_name"
+ ),
+ "ack": (
+ "build_job_namespace build_timeout_sec buildkit_address claim_timeout context dind_image extra_env "
+ "extra_volume_mounts extra_volumes image_pull_secret init_containers kubeconfig memory_limit_multiplier "
+ "namespace node_selector pod_annotations pod_capabilities_add pod_capabilities_drop pod_labels pod_overrides "
+ "pod_privileged pod_run_as_group pod_run_as_user registry sandbox_annotations sandbox_env_vars sandbox_image "
+ "sandbox_labels sandboxset_replicas service_account skip_image_check tolerations use_buildkit use_sandbox_claim"
+ ),
+ "openshift": "namespace service_account_name",
+ "novita": "dind_dockerd_start_cmd dind_template_alias",
+ "apple-container": "keep_containers",
+ "singularity": "singularity_force_pull singularity_image_cache_dir singularity_no_mount",
+ "islo": "delete_after_seconds gateway gateway_profile",
+ "tensorlake": "is_public preinstall_packages snapshot_id timeout_secs use_oci_image_build",
+ "cwsandbox": (
+ "base_url docker_image max_lifetime_seconds max_timeout_seconds mounts_json request_timeout_seconds secrets tags"
+ ),
+ "wandb": (
+ "base_url docker_image max_lifetime_seconds max_timeout_seconds mounts_json request_timeout_seconds secrets tags"
+ ),
+ "use-computer": (
+ "api_key base_url device_type family gateway_url host keepalive_interval mode override_exec_timeout platform "
+ "reservation_id resources runtime snapshot version"
+ ),
+ "cua-cloud": (
+ "api_key base_url bind_timeout_sec claim_spec claim_ttl_sec claims_path kubeconfig namespace "
+ "override_exec_timeout platform port_services ready_timeout_sec renew_interval startup_command sudo_password "
+ "svc_auth svc_suffix svc_url template token_url warmpool"
+ ),
+ "blaxel": "deployment_timeout_sec dind_extra_args dind_image region sandbox_version ttl",
+ "opensandbox": (
+ "api_key domain entrypoint extensions health_check_poll_interval_sec image_auth metadata protocol "
+ "ready_timeout_sec request_timeout_sec sandbox_timeout_sec skip_health_check use_server_proxy volumes"
+ ),
+ "beam": "keep_warm_seconds",
+ "skypilot": "context_name namespace platform pool registry secrets",
+ "hf-sandbox": "flavor forward_hf_token job_timeout",
+ "hyperbrowser": "image_id image_name region timeout_minutes",
+ "vercel": (
+ "builder_image create_timeout_sec credential_injection destroy_timeout_sec host_bootstrap host_image image "
+ "ports post_cancel_command_timeout_sec process_cancel_grace_sec process_wait_retry_delay_sec project_name "
+ "sandbox_lifetime_seconds task_bootstrap task_image transfer_timeout_sec"
+ ),
+}
+HARBOR_V022_ENVIRONMENT_KWARGS: dict[str, frozenset[str]] = {
+ mode: _HARBOR_V022_BASE_ENVIRONMENT_KWARGS | frozenset(names.split())
+ for mode, names in _HARBOR_V022_BACKEND_ENVIRONMENT_KWARGS.items()
+}
ENV_MODE_LOCAL = "local"
DEFAULT_ENV_MODE = "docker"
diff --git a/src/skillevaluator/utils/redaction.py b/src/skillevaluator/utils/redaction.py
index 33611c11..c87abd80 100644
--- a/src/skillevaluator/utils/redaction.py
+++ b/src/skillevaluator/utils/redaction.py
@@ -5,9 +5,11 @@
from __future__ import annotations
+import math
import re
from collections.abc import Iterable, Mapping
from typing import Any
+from urllib.parse import unquote
_SECRET_KEY_PARTS = {
"auth",
@@ -21,16 +23,38 @@
"secret",
"token",
}
+_PLURAL_SECRET_KEY_PARTS = {
+ "auths",
+ "authorizations",
+ "bearers",
+ "credentials",
+ "passwords",
+ "secrets",
+ "tokens",
+}
_TOKEN_COUNT_KEYS = {
+ "cached_tokens",
"completion_tokens",
+ "expected_max_tokens",
+ "frontmatter_tokens",
"input_tokens",
+ "instructions_tokens",
"n_input_tokens",
+ "n_cache_tokens",
"n_output_tokens",
"output_tokens",
"prompt_tokens",
+ "reasoning_output_tokens",
"last_token_usage",
+ "max_completion_tokens",
+ "max_output_tokens",
+ "max_tokens",
+ "recommended_max_tokens",
"token_count",
"tokens",
+ "total_cached_tokens",
+ "total_completion_tokens",
+ "total_prompt_tokens",
"total_tokens",
}
_SENSITIVE_KEY_PATTERN = (
@@ -38,6 +62,7 @@
r"access[_-]?key|session[_-]?token|private[_-]?key)[a-z0-9_.-]*"
)
_AUTH_HEADER_RE = re.compile(r"(?im)\b(?P(?:proxy-)?authorization)\s*:\s*(?P[A-Za-z]+)\s+[^\r\n]+")
+_CREDENTIAL_URI_USERINFO_RE = re.compile(r"(?i)(?P[a-z][a-z0-9+.-]{0,31}://)(?P[^\s/?#]+@)")
_SENSITIVE_QUOTED_ASSIGNMENT_RE = re.compile(
rf"(?i)\b(?P{_SENSITIVE_KEY_PATTERN})\s*(?P[:=])\s*(?P[\"'])(?P[^\r\n]*?)(?P=quote)"
)
@@ -76,10 +101,24 @@
),
(re.compile(r"(?:AKIA|ASIA)[A-Z0-9]{16}"), "aws-access-key-"),
(re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{8,}"), "Bearer "),
- (re.compile(r"(?"),
- (re.compile(r"(?"),
- (re.compile(r"(?"),
- (re.compile(r"(?"),
+ (re.compile(r"(?"),
+ (re.compile(r"nvapi-[a-zA-Z0-9_-]{8,}"), "nvapi-"),
+ (re.compile(r"crsr_[a-f0-9]{16,}"), "crsr_"),
+ (re.compile(r"sha256~[A-Za-z0-9._~-]+"), "sha256~"),
+ # GitHub's p/o/u/r families retain the 36-character opaque body. The s
+ # family also has a variable-length ``ghs_APPID_JWT`` stateless format.
+ (
+ re.compile(r"(?i)gh[pour]_[A-Za-z0-9]{36}"),
+ "github-token-",
+ ),
+ (
+ re.compile(r"(?i)ghs_[A-Za-z0-9.\-_]{36,}"),
+ "github-token-",
+ ),
+ (re.compile(r"(?i)github_pat_[A-Za-z0-9_]{20,}"), "github-token-"),
+ (re.compile(r"(?i)xox[baprs]-[A-Za-z0-9-]{10,}"), "slack-token-"),
+ (re.compile(r"AIza[A-Za-z0-9_-]{20,}"), "google-api-key-"),
+ (re.compile(r"(?i)glpat-[A-Za-z0-9_-]{20,}"), "gitlab-token-"),
)
@@ -102,7 +141,7 @@ def is_sensitive_key(key: str) -> bool:
return True
if "token" in parts or compact.endswith("token"):
return True
- return bool(parts & _SECRET_KEY_PARTS)
+ return bool(parts & (_SECRET_KEY_PARTS | _PLURAL_SECRET_KEY_PARTS))
def _redact_sensitive_assignment(match: re.Match[str]) -> str:
@@ -116,6 +155,40 @@ def _redact_auth_header(match: re.Match[str]) -> str:
return f"{match.group('key')}: {match.group('scheme')} "
+def credential_uri_secret_values(value: str, *, allow_schemeless: bool = False) -> set[str]:
+ """Return raw and decoded credential components from one URI authority."""
+ raw = str(value or "")
+ if not raw or "@" not in raw:
+ return set()
+ if "://" in raw:
+ _scheme, _separator, remainder = raw.partition("://")
+ elif allow_schemeless:
+ remainder = raw
+ else:
+ return set()
+ authority_end = min(
+ (index for delimiter in "/?#" if (index := remainder.find(delimiter)) >= 0),
+ default=len(remainder),
+ )
+ authority = remainder[:authority_end]
+ if "@" not in authority:
+ return set()
+ userinfo = authority.rsplit("@", 1)[0]
+ if not userinfo:
+ return set()
+
+ protected = {raw, userinfo}
+ decoded_userinfo = unquote(userinfo)
+ protected.add(decoded_userinfo)
+ for candidate in (userinfo, decoded_userinfo):
+ username, separator, password = candidate.partition(":")
+ if username:
+ protected.add(username)
+ if separator and password:
+ protected.add(password)
+ return {item for item in protected if item}
+
+
def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:
"""Best-effort masking for credentials before writing logs or artifacts."""
out = value
@@ -123,6 +196,8 @@ def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:
# or header rule can consume only its BEGIN delimiter and orphan the body.
for pattern, replacement in _PEM_REDACTIONS:
out = pattern.sub(replacement, out)
+ if "://" in out:
+ out = _CREDENTIAL_URI_USERINFO_RE.sub(r"\g@", out)
out = _AUTH_HEADER_RE.sub(_redact_auth_header, out)
out = _SENSITIVE_QUOTED_ASSIGNMENT_RE.sub(_redact_sensitive_assignment, out)
out = _SENSITIVE_COLON_ASSIGNMENT_RE.sub(_redact_sensitive_assignment, out)
@@ -136,15 +211,49 @@ def redact_sensitive_text(value: str, *, max_len: int | None = None) -> str:
return out
+def contains_credential_value(value: object) -> bool:
+ """Return whether text contains credential material, including embedded tokens."""
+ if value is None:
+ return False
+ text = str(value)
+ return bool(text) and redact_sensitive_text(text) != text
+
+
+def _is_finite_token_count(value: Any) -> bool:
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return False
+ return not isinstance(value, float) or math.isfinite(value)
+
+
+def _is_token_count_value(value: Any) -> bool:
+ if _is_finite_token_count(value):
+ return True
+ if not isinstance(value, Mapping) or not value:
+ return False
+ return all(
+ _normalized_key_parts(str(key))[0] in _TOKEN_COUNT_KEYS and _is_finite_token_count(item)
+ for key, item in value.items()
+ )
+
+
def redact_sensitive_data(value: Any, *, parent_key: str = "", max_str_len: int | None = None) -> Any:
"""Recursively redact structured data using secret-looking key names."""
+ normalized_parent, _parts = _normalized_key_parts(parent_key)
+ if normalized_parent in _TOKEN_COUNT_KEYS and not _is_token_count_value(value):
+ return ""
if is_sensitive_key(parent_key):
return ""
if isinstance(value, Mapping):
- return {
- str(key): redact_sensitive_data(item, parent_key=str(key), max_str_len=max_str_len)
- for key, item in value.items()
- }
+ redacted: dict[str, Any] = {}
+ for key, item in value.items():
+ raw_key = str(key)
+ # A credential can itself be used as a mapping key. Redacting the
+ # corresponding value is insufficient, and replacing the key can
+ # collapse distinct entries. Drop such entries collision-safely.
+ if redact_sensitive_text(raw_key, max_len=max_str_len) != raw_key:
+ continue
+ redacted[raw_key] = redact_sensitive_data(item, parent_key=raw_key, max_str_len=max_str_len)
+ return redacted
if isinstance(value, Iterable) and not isinstance(value, (str, bytes, bytearray, Mapping)):
return [redact_sensitive_data(item, parent_key=parent_key, max_str_len=max_str_len) for item in value]
if isinstance(value, str):
diff --git a/tests/golden/cli_surface.json b/tests/golden/cli_surface.json
index bc1aa820..35eacec6 100644
--- a/tests/golden/cli_surface.json
+++ b/tests/golden/cli_surface.json
@@ -256,7 +256,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -265,6 +268,14 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
"local"
],
"default": "docker",
@@ -274,6 +285,16 @@
],
"param_type": "option"
},
+ {
+ "multiple": true,
+ "name": "environment_kwarg",
+ "opts": [
+ "--ek",
+ "--environment-kwarg"
+ ],
+ "param_type": "option",
+ "type": "text"
+ },
{
"multiple": true,
"name": "agent_model",
@@ -324,7 +345,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -333,6 +357,14 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
"local"
],
"default": "docker",
@@ -342,6 +374,16 @@
],
"param_type": "option"
},
+ {
+ "multiple": true,
+ "name": "environment_kwarg",
+ "opts": [
+ "--ek",
+ "--environment-kwarg"
+ ],
+ "param_type": "option",
+ "type": "text"
+ },
{
"default": "False",
"is_flag": true,
@@ -587,7 +629,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -596,6 +641,14 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
"local"
],
"default": "docker",
@@ -604,6 +657,16 @@
"--env-mode"
],
"param_type": "option"
+ },
+ {
+ "multiple": true,
+ "name": "environment_kwarg",
+ "opts": [
+ "--ek",
+ "--environment-kwarg"
+ ],
+ "param_type": "option",
+ "type": "text"
}
]
},
@@ -1590,7 +1653,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -1599,6 +1665,14 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
"local"
],
"default": "docker",
@@ -1608,6 +1682,16 @@
],
"param_type": "option"
},
+ {
+ "multiple": true,
+ "name": "environment_kwarg",
+ "opts": [
+ "--ek",
+ "--environment-kwarg"
+ ],
+ "param_type": "option",
+ "type": "text"
+ },
{
"default": "False",
"is_flag": true,
@@ -2142,7 +2226,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -2151,6 +2238,14 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
"local"
],
"default": "docker",
@@ -2160,6 +2255,16 @@
],
"param_type": "option"
},
+ {
+ "multiple": true,
+ "name": "environment_kwarg",
+ "opts": [
+ "--ek",
+ "--environment-kwarg"
+ ],
+ "param_type": "option",
+ "type": "text"
+ },
{
"multiple": true,
"name": "agent_model",
@@ -2210,7 +2315,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -2219,6 +2327,14 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
"local"
],
"default": "docker",
@@ -2228,6 +2344,16 @@
],
"param_type": "option"
},
+ {
+ "multiple": true,
+ "name": "environment_kwarg",
+ "opts": [
+ "--ek",
+ "--environment-kwarg"
+ ],
+ "param_type": "option",
+ "type": "text"
+ },
{
"default": "False",
"is_flag": true,
@@ -2874,7 +3000,10 @@
"modal",
"runloop",
"langsmith",
+ "ec2",
"gke",
+ "ack",
+ "openshift",
"novita",
"apple-container",
"singularity",
@@ -2883,6 +3012,14 @@
"cwsandbox",
"wandb",
"use-computer",
+ "cua-cloud",
+ "blaxel",
+ "opensandbox",
+ "beam",
+ "skypilot",
+ "hf-sandbox",
+ "hyperbrowser",
+ "vercel",
"local"
],
"default": "docker",
@@ -2892,6 +3029,16 @@
],
"param_type": "option"
},
+ {
+ "multiple": true,
+ "name": "environment_kwarg",
+ "opts": [
+ "--ek",
+ "--environment-kwarg"
+ ],
+ "param_type": "option",
+ "type": "text"
+ },
{
"default": "False",
"is_flag": true,
diff --git a/tests/reporting/test_report_data_bounds.py b/tests/reporting/test_report_data_bounds.py
index 03b0c3fe..d32d809e 100644
--- a/tests/reporting/test_report_data_bounds.py
+++ b/tests/reporting/test_report_data_bounds.py
@@ -5,11 +5,16 @@
import json
import logging
+from datetime import UTC, datetime
from pathlib import Path
+from uuid import UUID
import pytest
+from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory
from skillevaluator.tier3.harbor import report_data
+from skillevaluator.tier3.harbor.collector import collect_harbor_results
+from skillevaluator.tier3.harbor.metrics import CUSTOM_ONLY_METRIC_SET, DEFAULT_METRIC_SET, DEFAULT_METRICS
def _write_summary(agent_dir: Path) -> None:
@@ -73,6 +78,213 @@ def test_normal_agent_artifacts_are_loaded_without_truncation_marker(tmp_path: P
assert "_report_truncation" not in agents["codex"]
+def test_report_loader_omits_unrepresentable_trajectory_token_counters(tmp_path: Path) -> None:
+ agent_dir = tmp_path / "codex"
+ _write_summary(agent_dir)
+ _write_trial(
+ agent_dir,
+ "case-001__1",
+ {"entry_id": "case-001", "accuracy": 1.0},
+ {
+ "steps": [{"action": "answer"}],
+ "final_metrics": {
+ "total_prompt_tokens": 10**400,
+ "total_completion_tokens": 4,
+ "total_cached_tokens": 1 << 53,
+ },
+ },
+ )
+
+ agents = report_data.load_agent_data(tmp_path)
+
+ assert agents["codex"]["rewards"][0]["_traj"] == {
+ "steps": 1,
+ "prompt_tokens": None,
+ "completion_tokens": 4,
+ "cached_tokens": None,
+ }
+
+
+def test_report_loader_marks_missing_trajectory_token_counters_unavailable(tmp_path: Path) -> None:
+ agent_dir = tmp_path / "codex"
+ _write_summary(agent_dir)
+ _write_trial(
+ agent_dir,
+ "case-001__1",
+ {"entry_id": "case-001", "accuracy": 1.0},
+ {"steps": [{"action": "answer"}]},
+ )
+
+ agents = report_data.load_agent_data(tmp_path)
+
+ assert agents["codex"]["rewards"][0]["_traj"] == {
+ "steps": 1,
+ "prompt_tokens": None,
+ "completion_tokens": None,
+ "cached_tokens": None,
+ }
+
+
+def test_report_loader_marks_missing_trajectory_steps_unavailable(tmp_path: Path) -> None:
+ agent_dir = tmp_path / "codex"
+ _write_summary(agent_dir)
+ _write_trial(
+ agent_dir,
+ "case-001__1",
+ {"entry_id": "case-001", "accuracy": 1.0},
+ {
+ "final_metrics": {
+ "total_prompt_tokens": 10,
+ "total_completion_tokens": 4,
+ }
+ },
+ )
+
+ agents = report_data.load_agent_data(tmp_path)
+
+ assert agents["codex"]["rewards"][0]["_traj"] == {
+ "steps": None,
+ "prompt_tokens": 10,
+ "completion_tokens": 4,
+ "cached_tokens": None,
+ }
+
+
+def test_actual_atif_v17_preserves_string_messages_token_metrics_and_provenance(tmp_path: Path) -> None:
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trajectories import Trajectory
+ from harbor.models.trial.result import TrialResult
+
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_dir = job_dir / "case-001__attempt"
+ agent_dir = trial_dir / "agent"
+ agent_dir.mkdir(parents=True)
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ trajectory = Trajectory.model_validate(
+ {
+ "schema_version": "ATIF-v1.7",
+ "session_id": "session-harbor-022",
+ "trajectory_id": "trajectory-harbor-022",
+ "agent": {
+ "name": "opencode",
+ "version": "test",
+ "model_name": "test-model",
+ "extra": {"adapter": "harbor-0.22"},
+ },
+ "steps": [
+ {
+ "step_id": 1,
+ "source": "user",
+ "message": "Run the requested evaluation.",
+ },
+ {
+ "step_id": 2,
+ "source": "agent",
+ "message": "The requested evaluation is complete.",
+ "llm_call_count": 1,
+ },
+ ],
+ "final_metrics": {
+ "total_prompt_tokens": 101,
+ "total_completion_tokens": 23,
+ "total_cached_tokens": 17,
+ "total_steps": 2,
+ "extra": {"producer_metric": "retained"},
+ },
+ "extra": {
+ "producer": "harbor-0.22",
+ "source_uri": "harbor://jobs/demo/trials/case-001__attempt",
+ },
+ }
+ )
+ (agent_dir / "trajectory.json").write_text(
+ trajectory.model_dump_json(indent=2, exclude_none=True),
+ encoding="utf-8",
+ )
+ trial_result = TrialResult.model_validate(
+ {
+ "id": UUID(int=2),
+ "task_name": "nvidia/skillevaluator-case-001",
+ "trial_name": trial_dir.name,
+ "trial_uri": trial_dir.as_uri(),
+ "task_id": {"path": str(job_dir / "task" / "case-001")},
+ "task_checksum": "harbor-0.22-atif-fixture",
+ "config": {
+ "task": {"path": str(job_dir / "task" / "case-001")},
+ "trial_name": trial_dir.name,
+ },
+ "agent_info": {
+ "name": "opencode",
+ "version": "test",
+ "model_info": {"name": "test-model"},
+ },
+ "agent_result": {
+ "n_input_tokens": 101,
+ "n_cache_tokens": 17,
+ "n_output_tokens": 23,
+ },
+ "verifier_result": {"rewards": {"overall": 1.0}},
+ "exception_info": None,
+ "started_at": now,
+ "finished_at": now,
+ "step_results": None,
+ }
+ )
+ job_result = JobResult(
+ id=UUID(int=1),
+ started_at=now,
+ updated_at=now,
+ finished_at=now,
+ n_total_trials=1,
+ stats=JobStats.from_trial_results([trial_result], n_total_trials=1),
+ trial_results=[trial_result],
+ )
+ (trial_dir / "result.json").write_text(trial_result.model_dump_json(indent=2), encoding="utf-8")
+ (job_dir / "result.json").write_text(job_result.model_dump_json(indent=2), encoding="utf-8")
+ results_dir = tmp_path / "results"
+
+ collected = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=results_dir,
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert collected["execution_status"] == "succeeded"
+ persisted_path = results_dir / "opencode" / "with-skill" / "trials" / trial_dir.name / "trajectory.json"
+ persisted = json.loads(persisted_path.read_text(encoding="utf-8"))
+ assert persisted["schema_version"] == "ATIF-v1.7"
+ assert persisted["session_id"] == "session-harbor-022"
+ assert persisted["trajectory_id"] == "trajectory-harbor-022"
+ assert persisted["agent"]["extra"] == {"adapter": "harbor-0.22"}
+ assert persisted["extra"] == {
+ "producer": "harbor-0.22",
+ "source_uri": "harbor://jobs/demo/trials/case-001__attempt",
+ }
+ assert [step["message"] for step in persisted["steps"]] == [
+ "Run the requested evaluation.",
+ "The requested evaluation is complete.",
+ ]
+ assert [(step["step_id"], step["source"]) for step in persisted["steps"]] == [
+ (1, "user"),
+ (2, "agent"),
+ ]
+ assert all(isinstance(step["message"], str) for step in persisted["steps"])
+
+ agents = report_data.load_agent_data(results_dir)
+ assert agents["opencode"]["rewards"][0]["_traj"] == {
+ "steps": 2,
+ "prompt_tokens": 101,
+ "completion_tokens": 23,
+ "cached_tokens": 17,
+ }
+
+
def test_agent_directory_symlink_is_not_discovered(tmp_path: Path) -> None:
outside_agent = tmp_path / "outside" / "codex"
_write_summary(outside_agent)
@@ -209,6 +421,181 @@ def test_excess_trials_are_capped_in_name_order(
assert any(reason["code"] == "trial_limit" and reason["limit"] == 2 for reason in _reasons(agents["codex"]))
+def test_report_uses_persisted_mixed_contract_flag_when_reward_sample_is_truncated(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(report_data, "_MAX_TRIALS_PER_CONDITION", 1)
+ run_dir = tmp_path / "results"
+ agent_dir = run_dir / "opencode"
+ summary_path = agent_dir / "with-skill" / "summary.json"
+ summary_path.parent.mkdir(parents=True)
+ summary_path.write_text(
+ json.dumps(
+ {
+ "scores": dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "custom_scores": {"domain_quality": 0.0},
+ "overall_score": 0.5,
+ "metric_set": DEFAULT_METRIC_SET,
+ "metrics": list(DEFAULT_METRICS),
+ "dimensions": {},
+ "num_trials": 2,
+ "num_reward_rows": 2,
+ "mixed_metric_contracts": True,
+ "pass_at_k": {},
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 2,
+ "scored_attempts": 2,
+ "job_failure": "",
+ "trial_failures": [],
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_trial(
+ agent_dir,
+ "a-standard",
+ {
+ "entry_id": "case-standard",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "overall": 1.0,
+ },
+ )
+ _write_trial(
+ agent_dir,
+ "z-custom",
+ {
+ "entry_id": "case-custom",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 0.0,
+ "custom_metrics": {"domain_quality": 0.0},
+ },
+ )
+
+ loaded = report_data.load_agent_data(run_dir)["opencode"]
+ skill_dir = tmp_path / "demo-skill"
+ skill_dir.mkdir()
+ result = agent_eval_result_from_directory(skill_dir, run_dir, use_llm_judge=False)
+
+ assert loaded["rewards_complete"] is False
+ assert [reward["entry_id"] for reward in loaded["rewards"]] == ["case-standard"]
+ assert loaded["mixed_metric_contracts_with_skill"] is True
+ assert result is not None
+ payload = result.metadata["agent_eval"]
+ assert payload["agents"]["opencode"]["with_skill"] == 0.5
+ assert payload["overall_score"] == 0.5
+
+
+def test_report_preserves_collector_declared_hidden_execution_error_count(tmp_path: Path) -> None:
+ run_dir = tmp_path / "results"
+ agent_dir = run_dir / "opencode"
+ summary_path = agent_dir / "with-skill" / "summary.json"
+ summary_path.parent.mkdir(parents=True)
+ summary_path.write_text(
+ json.dumps(
+ {
+ "scores": {},
+ "metrics": [],
+ "overall_score": 0.0,
+ "num_trials": 0,
+ "num_reward_rows": 0,
+ "pass_at_k": {},
+ "execution_status": "failed",
+ "execution_errors": ["visible collector diagnostic"],
+ "execution_error_details_total": 300,
+ "execution_error_details_shown": 1,
+ "execution_error_details_truncated": True,
+ "expected_attempts": 300,
+ "scored_attempts": 0,
+ "job_failure": "",
+ "trial_failures": [],
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ loaded = report_data.load_agent_data(run_dir)["opencode"]
+ skill_dir = tmp_path / "demo-skill"
+ skill_dir.mkdir()
+ result = agent_eval_result_from_directory(skill_dir, run_dir, use_llm_judge=False)
+
+ assert loaded["conditions"]["with_skill"]["execution_error_details_total"] == 300
+ assert loaded["conditions"]["with_skill"]["execution_error_details_shown"] == 1
+ assert loaded["conditions"]["with_skill"]["execution_error_details_truncated"] is True
+ assert loaded["execution_error_details_total"] == 300
+ assert loaded["execution_error_details_shown"] == 1
+ assert loaded["execution_error_details_truncated"] is True
+ assert result is not None
+ payload = result.metadata["agent_eval"]
+ assert payload["agents"]["opencode"]["execution_error_details_total"] == 300
+ assert payload["agents"]["opencode"]["execution_error_details_shown"] == 1
+ assert payload["agents"]["opencode"]["execution_error_details_truncated"] is True
+ assert payload["summary"]["execution_error_details_total"] == 300
+ assert payload["summary"]["execution_error_details_shown"] == 1
+ assert payload["summary"]["execution_error_details_truncated"] is True
+ assert payload["execution_error_details_total"] == 300
+ assert payload["execution_error_details_shown"] == 1
+ assert payload["execution_error_details_truncated"] is True
+
+
+def test_legacy_mixed_contract_summary_without_flag_uses_reward_inference(tmp_path: Path) -> None:
+ run_dir = tmp_path / "results"
+ agent_dir = run_dir / "opencode"
+ summary_path = agent_dir / "with-skill" / "summary.json"
+ summary_path.parent.mkdir(parents=True)
+ summary_path.write_text(
+ json.dumps(
+ {
+ "scores": dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "custom_scores": {"domain_quality": 0.0},
+ "overall_score": 0.5,
+ "metric_set": DEFAULT_METRIC_SET,
+ "metrics": list(DEFAULT_METRICS),
+ "dimensions": {},
+ "num_trials": 2,
+ "num_reward_rows": 2,
+ "pass_at_k": {},
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 2,
+ "scored_attempts": 2,
+ "job_failure": "",
+ "trial_failures": [],
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_trial(
+ agent_dir,
+ "case-standard",
+ {
+ "entry_id": "case-standard",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "overall": 1.0,
+ },
+ )
+ _write_trial(
+ agent_dir,
+ "case-custom",
+ {
+ "entry_id": "case-custom",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 0.0,
+ "custom_metrics": {"domain_quality": 0.0},
+ },
+ )
+ skill_dir = tmp_path / "demo-skill"
+ skill_dir.mkdir()
+
+ result = agent_eval_result_from_directory(skill_dir, run_dir, use_llm_judge=False)
+
+ assert result is not None
+ assert result.metadata["agent_eval"]["agents"]["opencode"]["with_skill"] == 0.5
+
+
def test_staged_tasks_and_dataset_records_are_capped_deterministically(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/reporting/test_unified_tier3_report.py b/tests/reporting/test_unified_tier3_report.py
index 832d0f84..0188954d 100644
--- a/tests/reporting/test_unified_tier3_report.py
+++ b/tests/reporting/test_unified_tier3_report.py
@@ -15,6 +15,7 @@
_attach_harbor_evidence_to_conclusions,
_attach_harbor_evidence_to_recommendations,
_attach_harbor_evidence_to_suggestions_v2,
+ _cases,
_evaluator_cards,
_metric_evidence,
_normalize_trials,
@@ -29,8 +30,9 @@
from skillevaluator.reporting import html as html_module
from skillevaluator.reporting.html import PackageLoader, _compact_json
from skillevaluator.tier3.harbor.collector import _paired_pass_comparison, _wilson_score_interval
-from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS
-from skillevaluator.tier3.harbor.report_data import build_dataset_snapshot
+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS, MAX_CUSTOM_METRIC_NAME_BYTES
+from skillevaluator.tier3.harbor.report import _extract_findings
+from skillevaluator.tier3.harbor.report_data import build_dataset_snapshot, metrics_for_agents
def _write_summary(
@@ -193,6 +195,261 @@ def test_normalize_trials_preserves_only_trusted_standard_invocation_evidence()
assert "invocation_evidence_source" not in trial
+def test_normalize_trials_collapses_multi_step_rows_to_one_logical_attempt() -> None:
+ [trial] = _normalize_trials(
+ [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "metric_set": "skill-evaluator-default-v2",
+ **dict.fromkeys(DEFAULT_METRICS, 0.2),
+ "overall": 0.2,
+ "_traj": {"steps": 2, "prompt_tokens": 10},
+ "skill_invoked": True,
+ "invocation_evidence_source": "trajectory",
+ "warnings": ["first warning"],
+ },
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "metric_set": "skill-evaluator-default-v2",
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "overall": 1.0,
+ "_traj": {"steps": 3, "prompt_tokens": 20},
+ "skill_invoked": False,
+ "invocation_evidence_source": "trajectory",
+ "warnings": ["second warning"],
+ },
+ ],
+ ["accuracy"],
+ )
+
+ assert trial["trial_id"] == "case-1__attempt1"
+ assert trial["entry_id"] == "case-1"
+ assert trial["scores"] == {"accuracy": 0.6}
+ assert trial["overall"] == 0.6
+ assert trial["warnings"] == ["first warning", "second warning"]
+ # Step trajectories may be independent or cumulative. Without the merged
+ # root trajectory, omit counters rather than double-counting resume data.
+ assert "steps" not in trial
+ assert "tokens" not in trial
+ assert "skill_invoked" not in trial
+ assert "invocation_evidence_source" not in trial
+
+
+def test_normalize_trials_omits_unavailable_tokens_but_preserves_steps() -> None:
+ missing, available = _normalize_trials(
+ [
+ {
+ "trial_id": "missing",
+ "entry_id": "missing",
+ "overall": 1.0,
+ "_traj": {
+ "steps": 4,
+ "prompt_tokens": None,
+ "completion_tokens": None,
+ "cached_tokens": None,
+ },
+ },
+ {
+ "trial_id": "available",
+ "entry_id": "available",
+ "overall": 1.0,
+ "_traj": {
+ "steps": 3,
+ "prompt_tokens": 10,
+ "completion_tokens": 4,
+ "cached_tokens": None,
+ },
+ },
+ ],
+ [],
+ )
+
+ assert missing["steps"] == 4
+ assert "tokens" not in missing
+ assert available["steps"] == 3
+ assert available["tokens"] == {"prompt": 10, "completion": 4}
+
+
+def test_normalize_trials_omits_unavailable_steps_but_preserves_tokens() -> None:
+ [trial] = _normalize_trials(
+ [
+ {
+ "trial_id": "missing-steps",
+ "entry_id": "missing-steps",
+ "overall": 1.0,
+ "_traj": {
+ "steps": None,
+ "prompt_tokens": 10,
+ "completion_tokens": 4,
+ "cached_tokens": 2,
+ },
+ }
+ ],
+ [],
+ )
+
+ assert "steps" not in trial
+ assert trial["tokens"] == {"prompt": 10, "completion": 4, "cached": 2}
+
+
+def test_normalize_trials_fails_closed_when_a_grouped_standard_row_is_incomplete() -> None:
+ complete = {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "metric_set": "skill-evaluator-default-v2",
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ }
+ incomplete = {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "metric_set": "skill-evaluator-default-v2",
+ "accuracy": 0.0,
+ }
+
+ [trial] = _normalize_trials([complete, incomplete], list(DEFAULT_METRICS))
+
+ assert trial["scores"] == {"accuracy": 0.5}
+ assert trial["overall"] is None
+ assert _cases({"rewards": [complete, incomplete]}) == [{"entry_id": "case-1", "overall": None}]
+
+
+def test_normalize_trials_fails_closed_for_conflicting_grouped_entry_identities() -> None:
+ rows = [
+ {
+ "trial_id": "shared-attempt",
+ "entry_id": entry_id,
+ "metric_set": "skill-evaluator-default-v2",
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ }
+ for entry_id in ("case-1", "case-2")
+ ]
+
+ [trial] = _normalize_trials(rows, list(DEFAULT_METRICS))
+
+ assert trial["overall"] is None
+ assert _cases({"rewards": rows}) == [{"entry_id": "case-1", "overall": None}]
+
+
+@pytest.mark.parametrize("reverse_rows", [False, True])
+def test_normalize_trials_uses_group_wide_metric_precedence_for_mixed_step_verifiers(
+ reverse_rows: bool,
+) -> None:
+ rows = [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "metric_set": "custom-only",
+ **dict.fromkeys(DEFAULT_METRICS, 0.0),
+ "metrics": {
+ "domain_quality": {"score": 0.2},
+ "skill_execution": {"score": 0.0},
+ },
+ "details": {"skill_execution": {"reason": "spoofed canonical evidence"}},
+ "overall": 0.2,
+ },
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "metric_set": "skill-evaluator-default-v2",
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ },
+ ]
+ if reverse_rows:
+ rows.reverse()
+
+ [trial] = _normalize_trials(rows, list(DEFAULT_METRICS))
+
+ assert trial["scores"] == dict.fromkeys(DEFAULT_METRICS, 1.0)
+ assert trial["overall"] == 0.6
+ assert _cases({"rewards": rows}) == [{"entry_id": "case-1", "overall": 0.6}]
+ assert _metric_evidence("skill_execution", rows, _ReportBudget()) == []
+ assert _extract_findings(rows, canonical_scores=dict.fromkeys(DEFAULT_METRICS, 1.0)) == []
+
+
+def test_report_metric_discovery_honors_explicit_custom_only_declaration() -> None:
+ assert (
+ metrics_for_agents(
+ {
+ "codex": {
+ "with_skill": {},
+ "rewards": [
+ {
+ "metric_set": "custom-only",
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "overall": 0.5,
+ }
+ ],
+ }
+ }
+ )
+ == []
+ )
+
+
+def test_best_agent_compares_standard_and_custom_only_agent_overall_scores() -> None:
+ payload = build_agent_eval_payload(
+ "mixed-agent-contracts",
+ {
+ "standard-agent": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": dict.fromkeys(DEFAULT_METRICS, 0.8),
+ "metrics_with_skill": list(DEFAULT_METRICS),
+ "overall_with_skill": 0.8,
+ "rewards": [
+ {
+ "entry_id": "standard-case",
+ "metric_set": "skill-evaluator-default-v2",
+ **dict.fromkeys(DEFAULT_METRICS, 0.8),
+ "overall": 0.8,
+ }
+ ],
+ },
+ "custom-agent": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": {},
+ "metrics_with_skill": [],
+ "overall_with_skill": 0.9,
+ "without_skill": {},
+ "metrics_without_skill": [],
+ "overall_without_skill": 0.4,
+ "rewards": [
+ {
+ "entry_id": "custom-case",
+ "metric_set": "custom-only",
+ "overall": 0.9,
+ "custom_metrics": {"task_quality": 0.9},
+ }
+ ],
+ "rewards_baseline": [
+ {
+ "entry_id": "custom-case",
+ "metric_set": "custom-only",
+ "overall": 0.4,
+ "custom_metrics": {"task_quality": 0.4},
+ }
+ ],
+ },
+ },
+ use_llm_judge=False,
+ )
+
+ assert payload is not None
+ assert payload["agents"]["standard-agent"]["with_skill"] == 0.8
+ assert payload["agents"]["custom-agent"]["with_skill"] == 0.9
+ assert payload["agents"]["custom-agent"]["baseline"] == 0.4
+ assert payload["agents"]["custom-agent"]["lift"] == 0.5
+ assert payload["best_agent"] == "custom-agent"
+ assert payload["overall_score"] == 0.9
+
+
def test_case_grounded_harbor_links_win_over_positional_fallback() -> None:
evidence_links = [
{"url": "https://harbor.example.test/case-1", "entry_id": "case-1", "label": "case-1"},
@@ -282,6 +539,222 @@ def test_standalone_tier3_uses_generic_tier3_only_report(tmp_path: Path) -> None
assert "SkillEvaluator" in html
+def test_trial_report_renders_unavailable_tokens_as_unavailable() -> None:
+ payload = build_agent_eval_payload(
+ "demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": {"accuracy": 1.0},
+ "rewards": [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "accuracy": 1.0,
+ "_traj": {
+ "steps": 4,
+ "prompt_tokens": None,
+ "completion_tokens": None,
+ "cached_tokens": None,
+ },
+ }
+ ],
+ }
+ },
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ trials_html = _tier3_page(_render_agent_payload(payload), "trials", "agents")
+
+ assert "With-Skill Steps per Eval Case" in trials_html
+ assert "With-Skill Token Usage by Agent" not in trials_html
+ assert "With-Skill Per-Evaluator Drill-Down" in trials_html
+ assert re.search(r"4 \s*— ", trials_html)
+
+
+def test_trial_report_omits_token_chart_for_incomplete_agent_telemetry() -> None:
+ payload = build_agent_eval_payload(
+ "demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "expected_attempts": 2,
+ "scored_attempts": 2,
+ "with_skill": {"accuracy": 1.0},
+ "rewards": [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "accuracy": 1.0,
+ "_traj": {
+ "steps": 4,
+ "prompt_tokens": 10,
+ "completion_tokens": 4,
+ "cached_tokens": None,
+ },
+ },
+ {
+ "trial_id": "case-2__attempt1",
+ "entry_id": "case-2",
+ "accuracy": 1.0,
+ "_traj": {
+ "steps": 3,
+ "prompt_tokens": None,
+ "completion_tokens": None,
+ "cached_tokens": None,
+ },
+ },
+ ],
+ }
+ },
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ trials_html = _tier3_page(_render_agent_payload(payload), "trials", "agents")
+
+ assert "With-Skill Token Usage by Agent" not in trials_html
+ assert 'id="tier3-token-chart"' not in trials_html
+ assert re.search(r"4 \s*14 ", trials_html)
+ assert re.search(r"3 \s*— ", trials_html)
+
+
+def test_trial_report_omits_token_chart_when_any_agent_telemetry_is_incomplete() -> None:
+ payload = build_agent_eval_payload(
+ "demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": {"accuracy": 1.0},
+ "rewards": [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "accuracy": 1.0,
+ "_traj": {
+ "steps": 4,
+ "prompt_tokens": 10,
+ "completion_tokens": 4,
+ "cached_tokens": 2,
+ },
+ }
+ ],
+ },
+ "opencode": {
+ "execution_status": "succeeded",
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": {"accuracy": 1.0},
+ "rewards": [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "accuracy": 1.0,
+ "_traj": {
+ "steps": 3,
+ "prompt_tokens": None,
+ "completion_tokens": None,
+ "cached_tokens": None,
+ },
+ }
+ ],
+ },
+ },
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ trials_html = _tier3_page(_render_agent_payload(payload), "trials", "agents")
+
+ assert "With-Skill Token Usage by Agent" not in trials_html
+ assert 'id="tier3-token-chart"' not in trials_html
+
+
+def test_trial_report_omits_token_chart_when_reported_agent_has_no_trials() -> None:
+ payload = build_agent_eval_payload(
+ "demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": {"accuracy": 1.0},
+ "rewards": [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "accuracy": 1.0,
+ "_traj": {
+ "steps": 4,
+ "prompt_tokens": 10,
+ "completion_tokens": 4,
+ "cached_tokens": 2,
+ },
+ }
+ ],
+ },
+ "opencode": {
+ "execution_status": "failed",
+ "execution_errors": ["malformed judge response"],
+ "expected_attempts": 1,
+ "scored_attempts": 0,
+ "with_skill": {},
+ "rewards": [],
+ },
+ },
+ use_llm_judge=False,
+ )
+ assert payload is not None
+ assert set(payload["agents"]) == {"codex", "opencode"}
+
+ trials_html = _tier3_page(_render_agent_payload(payload), "trials", "agents")
+
+ assert "With-Skill Token Usage by Agent" not in trials_html
+ assert 'id="tier3-token-chart"' not in trials_html
+
+
+def test_trial_charts_preserve_token_and_step_counter_semantics() -> None:
+ payload = build_agent_eval_payload(
+ "demo",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": {"accuracy": 1.0},
+ "rewards": [
+ {
+ "trial_id": "case-1__attempt1",
+ "entry_id": "case-1",
+ "accuracy": 1.0,
+ "_traj": {
+ "steps": 4,
+ "prompt_tokens": 10,
+ "completion_tokens": 4,
+ "cached_tokens": 2,
+ },
+ }
+ ],
+ }
+ },
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ html = _render_agent_payload(payload)
+ trials_html = _tier3_page(html, "trials", "agents")
+
+ assert 'id="tier3-token-chart"' in trials_html
+ assert '{ label: "Prompt (total)"' in html
+ assert '{ label: "Cached (included in prompt)"' in html
+ assert "if (observed.length !== matching.length) return null;" in html
+
+
def test_authenticated_pre_status_rerender_preserves_historical_scores(tmp_path: Path) -> None:
skill = tmp_path / "demo"
skill.mkdir()
@@ -661,6 +1134,228 @@ def items(self):
assert budget.omitted["evaluator_cards"] > 0
+def test_findings_prefer_custom_details_only_for_custom_metric_collisions() -> None:
+ reward = {
+ "entry_id": "case-1",
+ "metric_set": "skill-evaluator-default-v2",
+ "security": 0.9,
+ "custom_metrics": {"domain_quality": 0.8},
+ "details": {
+ "security": {"reason": "standard evidence"},
+ "domain_quality": {"reason": "ordinary collision"},
+ },
+ "custom_details": {
+ "security": {"reason": "custom collision"},
+ "domain_quality": {"reason": "custom evidence"},
+ },
+ }
+
+ findings = _extract_findings(
+ [reward],
+ canonical_scores={"security": 0.9, "domain_quality": 0.8},
+ )
+
+ by_metric = {finding["metric"]: finding for finding in findings}
+ assert by_metric["security"]["reasons"] == ["standard evidence"]
+ assert by_metric["domain_quality"]["reasons"] == ["custom evidence"]
+
+
+def test_unified_evidence_prefers_custom_details_only_for_custom_metric_collisions() -> None:
+ reward = {
+ "entry_id": "case-1",
+ "metric_set": "skill-evaluator-default-v2",
+ "security": 0.9,
+ "custom_metrics": {"domain_quality": 0.8},
+ "details": {
+ "security": {"reason": "standard evidence"},
+ "domain_quality": {"reason": "ordinary collision"},
+ },
+ "custom_details": {
+ "security": {"reason": "custom collision"},
+ "domain_quality": {"reason": "custom evidence"},
+ },
+ }
+
+ standard = _metric_evidence("security", [reward], _ReportBudget())
+ custom = _metric_evidence("domain_quality", [reward], _ReportBudget())
+
+ assert standard[0]["notes"] == ["standard evidence"]
+ assert custom[0]["notes"] == ["custom evidence"]
+
+
+def test_unified_evidence_does_not_fallback_from_explicit_malformed_custom_detail() -> None:
+ reward = {
+ "entry_id": "case-1",
+ "metric_set": "custom-only",
+ "custom_metrics": {"domain_quality": 0.8},
+ "details": {"domain_quality": {"reason": "ordinary collision"}},
+ "custom_details": {"domain_quality": None},
+ }
+
+ evidence = _metric_evidence("domain_quality", [reward], _ReportBudget())
+
+ assert evidence == []
+
+
+def test_unified_evidence_keeps_legacy_custom_detail_fallback_when_custom_key_is_absent() -> None:
+ reward = {
+ "entry_id": "case-1",
+ "metric_set": "custom-only",
+ "custom_metrics": {"domain_quality": 0.8},
+ "details": {"domain_quality": {"reason": "legacy custom evidence"}},
+ "custom_details": {"another_metric": {"reason": "unrelated"}},
+ }
+
+ evidence = _metric_evidence("domain_quality", [reward], _ReportBudget())
+
+ assert evidence[0]["notes"] == ["legacy custom evidence"]
+
+
+@pytest.mark.parametrize("source", ["configured", "reward"])
+def test_custom_evaluator_cards_reject_scores_outside_the_unit_interval(source: str) -> None:
+ custom_with_skill = {"domain_quality": 1e308} if source == "configured" else {}
+ rewards = (
+ [
+ {
+ "metric_set": "custom-only",
+ "overall": 0.5,
+ "custom_metrics": {"domain_quality": 1e308},
+ }
+ ]
+ if source == "reward"
+ else []
+ )
+
+ cards = _evaluator_cards(
+ {},
+ rewards=rewards,
+ custom_with_skill=custom_with_skill,
+ custom_without_skill={},
+ custom_lift={},
+ report_budget=_ReportBudget(),
+ )
+
+ assert not any(card["id"] == "domain_quality" for card in cards)
+
+
+def test_legacy_summary_custom_metric_names_follow_current_publication_policy(tmp_path: Path) -> None:
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ run_dir = tmp_path / "results" / "legacy-custom-metrics"
+ summary = run_dir / "opencode" / "with-skill" / "summary.json"
+ summary.parent.mkdir(parents=True)
+ credential_name = "ghp_" + ("a" * 36)
+ credential_field = "api_key_quality"
+ oversized_name = ("é" * (MAX_CUSTOM_METRIC_NAME_BYTES // 2)) + "x"
+ assert len(oversized_name.encode("utf-8")) == MAX_CUSTOM_METRIC_NAME_BYTES + 1
+ trial_reward = summary.parent / "trials" / "case-001" / "reward.json"
+ trial_reward.parent.mkdir(parents=True)
+ trial_reward.write_text(
+ json.dumps(
+ {
+ "entry_id": "case-001",
+ "metric_set": "custom-only",
+ "overall": 0.8,
+ "custom_metrics": {
+ credential_name: 0.5,
+ credential_field: 0.6,
+ oversized_name: 0.7,
+ "domain_quality": 0.8,
+ },
+ credential_name: 0.5,
+ credential_field: 0.6,
+ oversized_name: 0.7,
+ "domain_quality": 0.8,
+ }
+ ),
+ encoding="utf-8",
+ )
+ summary.write_text(
+ json.dumps(
+ {
+ "scores": {},
+ "custom_scores": {
+ credential_name: 0.5,
+ credential_field: 0.6,
+ oversized_name: 0.7,
+ "domain_quality": 0.8,
+ },
+ "overall_score": 0.8,
+ "metrics": [],
+ "num_trials": 1,
+ "num_reward_rows": 1,
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ result = agent_eval_result_from_directory(skill, run_dir, use_llm_judge=False)
+
+ assert result is not None
+ payload = result.metadata["agent_eval"]
+ custom_card_ids = [
+ card["id"] for card in payload["agents"]["opencode"]["evaluator_cards"] if card["label"].startswith("Custom: ")
+ ]
+ assert custom_card_ids == ["domain_quality"]
+ [raw_reward] = payload["provenance"]["raw_trial_rewards"]["opencode"]
+ assert raw_reward["entry_id"] == "case-001"
+ assert raw_reward["metric_set"] == "custom-only"
+ assert raw_reward["overall"] == 0.8
+ assert raw_reward["custom_metrics"] == {"domain_quality": 0.8}
+ assert raw_reward["domain_quality"] == 0.8
+ serialized = json.dumps(payload)
+ assert credential_name not in serialized
+ assert credential_field not in serialized
+ assert oversized_name not in serialized
+
+
+def test_legacy_non_mapping_custom_scores_and_lift_degrade_safely(tmp_path: Path) -> None:
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ run_dir = tmp_path / "results" / "legacy-malformed-custom-maps"
+ agent_dir = run_dir / "opencode"
+
+ for variant, overall, custom_scores in (
+ ("with-skill", 0.8, {"domain_quality": 0.8}),
+ ("without-skill", 0.4, "not-a-custom-score-map"),
+ ):
+ summary = agent_dir / variant / "summary.json"
+ summary.parent.mkdir(parents=True)
+ summary.write_text(
+ json.dumps(
+ {
+ "scores": {},
+ "custom_scores": custom_scores,
+ "overall_score": overall,
+ "metrics": [],
+ "num_trials": 0,
+ "num_reward_rows": 0,
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 0,
+ "scored_attempts": 0,
+ }
+ ),
+ encoding="utf-8",
+ )
+ (agent_dir / "custom_lift.json").write_text(json.dumps("not-a-custom-lift-map"), encoding="utf-8")
+
+ result = agent_eval_result_from_directory(skill, run_dir, use_llm_judge=False)
+
+ assert result is not None
+ payload = result.metadata["agent_eval"]
+ [custom_card] = [
+ card for card in payload["agents"]["opencode"]["evaluator_cards"] if card["id"] == "domain_quality"
+ ]
+ assert custom_card["with_skill"] == 0.8
+ assert custom_card["baseline"] is None
+ assert custom_card["lift"] is None
+
+
def test_raw_reward_projection_bulk_stops_when_field_budget_is_full() -> None:
class ExplodingReward(dict[str, float]):
def items(self):
@@ -876,6 +1571,89 @@ def test_non_finite_report_numbers_are_sanitized_before_canonical_json() -> None
assert _embedded_tier3_payload(html)["dataset"][0]["score"] is None
+def test_canonical_payload_sanitizes_unsafe_integers_for_browser_json_semantics() -> None:
+ safe_integer_metadata = {
+ "max_safe": (1 << 53) - 1,
+ "min_safe": -((1 << 53) - 1),
+ "enabled": True,
+ }
+ unsafe_integer_metadata = {
+ **safe_integer_metadata,
+ "too_large": 10**400,
+ "too_small": -(10**400),
+ }
+ payload = build_agent_eval_payload(
+ "browser-safe-json",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": {"security": 1.0},
+ "rewards": [
+ {
+ "entry_id": "case-1",
+ "security": 1.0,
+ "overall": 1.0,
+ "metadata": unsafe_integer_metadata,
+ }
+ ],
+ }
+ },
+ dataset=[{"id": "case-1", "metadata": unsafe_integer_metadata}],
+ use_llm_judge=False,
+ )
+ assert payload is not None
+
+ canonical_metadata = (
+ payload["dataset"][0]["metadata"],
+ payload["provenance"]["raw_trial_rewards"]["codex"][0]["metadata"],
+ )
+ for metadata in canonical_metadata:
+ assert metadata["too_large"] is None
+ assert metadata["too_small"] is None
+ assert metadata["max_safe"] == (1 << 53) - 1
+ assert type(metadata["max_safe"]) is int
+ assert metadata["min_safe"] == -((1 << 53) - 1)
+ assert type(metadata["min_safe"]) is int
+ assert metadata["enabled"] is True
+
+ encoded = json.dumps(payload, allow_nan=False)
+ browser_payload = json.loads(encoded, parse_int=float)
+ dataset_metadata = browser_payload["dataset"][0]["metadata"]
+ raw_metadata = browser_payload["provenance"]["raw_trial_rewards"]["codex"][0]["metadata"]
+
+ for metadata in (dataset_metadata, raw_metadata):
+ assert metadata["too_large"] is None
+ assert metadata["too_small"] is None
+ assert metadata["max_safe"] == float((1 << 53) - 1)
+ assert metadata["min_safe"] == -float((1 << 53) - 1)
+ assert metadata["enabled"] is True
+
+
+@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), float("-inf"), 10**400])
+def test_invalid_legacy_attempt_threshold_does_not_crash_canonical_report(invalid: float | int) -> None:
+ payload = build_agent_eval_payload(
+ "invalid-policy-number",
+ {
+ "codex": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "with_skill": dict.fromkeys(DEFAULT_METRICS, 1.0),
+ }
+ },
+ attempt_policy={"pass_threshold": invalid},
+ use_llm_judge=False,
+ )
+
+ assert payload is not None
+ assert payload["verdict_policy"]["attempt_pass_threshold"] is None
+ json.dumps(payload, allow_nan=False)
+
+
def test_canonical_html_serializer_rejects_non_finite_numbers() -> None:
with pytest.raises(ValueError, match="Out of range float values"):
_compact_json({"score": float("nan")})
diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py
index e92a708d..a209fc14 100644
--- a/tests/test_ci_workflows.py
+++ b/tests/test_ci_workflows.py
@@ -4,6 +4,7 @@
from __future__ import annotations
import re
+import tomllib
from pathlib import Path
from typing import Any
@@ -11,6 +12,7 @@
ROOT = Path(__file__).resolve().parents[1]
WORKFLOWS = ROOT / ".github" / "workflows"
+GITLEAKS_CONFIG = ROOT / ".gitleaks.toml"
REQUIRED_CI_JOBS = {
"test-python-312": "Tests (Python 3.12)",
@@ -134,6 +136,7 @@ def test_full_lane_keeps_the_existing_commands_and_runners() -> None:
assert jobs["tier2-windows"]["runs-on"] == "windows-latest"
assert "tests/embedding" in _runs(jobs["tier2-macos"])
assert "tests/embedding" in _runs(jobs["tier2-windows"])
+ assert "tests/test_harbor_runner_status.py" in _runs(jobs["tier2-windows"])
assert jobs["tier3-macos"]["runs-on"] == "macos-latest"
assert "tests/test_tier3_progress.py" in _runs(jobs["tier3-macos"])
assert jobs["native-windows-local-mode"]["runs-on"] == "windows-latest"
@@ -174,6 +177,31 @@ def test_security_keeps_gitleaks_always_on_and_skips_only_nonessential_jobs() ->
assert "vars.ENABLE_GITHUB_ADVANCED_SECURITY == 'true'" in codeql_if
+def test_gitleaks_synthetic_harbor_allowlists_are_exactly_scoped() -> None:
+ config = tomllib.loads(GITLEAKS_CONFIG.read_text(encoding="utf-8"))
+ allowlists = {entry["description"]: entry for entry in config["allowlists"]}
+
+ expected = {
+ "Synthetic NVIDIA Build token used by streaming redaction tests": (
+ "^Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8$",
+ r"^tests/test_harbor_local_mode\.py$",
+ ),
+ "Synthetic main-container values used by Docker isolation tests": (
+ r"^main-(persistent|task|scoped)-secret-[0-9]{5}$",
+ r"^tests/test_harbor_secure_docker_environment\.py$",
+ ),
+ }
+ for description, (regex, path) in expected.items():
+ entry = allowlists[description]
+ assert entry == {
+ "description": description,
+ "condition": "AND",
+ "targetRules": ["generic-api-key"],
+ "regexes": [regex],
+ "paths": [path],
+ }
+
+
def test_dco_stays_unconditional_and_has_no_path_filter() -> None:
dco = _load("dco.yml")
@@ -200,6 +228,8 @@ def test_changed_workflows_pin_every_action_to_a_commit() -> None:
def test_changed_workflows_do_not_persist_checkout_credentials() -> None:
for workflow_name in ("ci.yml", "security.yml"):
- checkout_steps = [step for step in _all_steps(_load(workflow_name)) if step.get("uses", "").startswith("actions/checkout@")]
+ checkout_steps = [
+ step for step in _all_steps(_load(workflow_name)) if step.get("uses", "").startswith("actions/checkout@")
+ ]
assert checkout_steps
assert all(step.get("with", {}).get("persist-credentials") == "false" for step in checkout_steps)
diff --git a/tests/test_evaluation_service.py b/tests/test_evaluation_service.py
index b884bfc0..6a153164 100644
--- a/tests/test_evaluation_service.py
+++ b/tests/test_evaluation_service.py
@@ -139,16 +139,120 @@ def _fake_evaluate(self, options: EvaluationOptions, *, progress_reporter=None)
monkeypatch.setattr(EvaluationService, "evaluate", _fake_evaluate, raising=True)
result = CliRunner().invoke(
- cli, ["evaluate", str(FIXTURE), "-a", "codex", "--env-mode", "docker", "--skip-baseline"]
+ cli,
+ [
+ "evaluate",
+ str(FIXTURE),
+ "-a",
+ "codex",
+ "--env-mode",
+ "ec2",
+ "--environment-kwarg",
+ "region=us-west-2",
+ "--ek",
+ "launch_mode=attach",
+ "--ek",
+ "instance_id=i-123",
+ "--skip-baseline",
+ ],
)
assert result.exit_code == 0, result.output
opts = captured["options"]
assert isinstance(opts, EvaluationOptions)
assert opts.agents == "codex"
- assert opts.env_mode == "docker"
+ assert opts.env_mode == "ec2"
+ assert opts.environment_kwarg == (
+ "region=us-west-2",
+ "launch_mode=attach",
+ "instance_id=i-123",
+ )
assert opts.skip_baseline is True
+def test_cli_validate_forwards_environment_kwargs_to_tier3(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skillevaluator import cli as cli_module
+ from skillevaluator.models.result import ValidationResult
+
+ captured: dict[str, object] = {}
+
+ def _tier1(*_args: object, **_kwargs: object) -> list[ValidationResult]:
+ result = ValidationResult(validator_name="SCHEMA")
+ result.add_success("schema", "ok")
+ return [result]
+
+ def _tier3(*_args: object, **kwargs: object) -> ValidationResult:
+ captured.update(kwargs)
+ result = ValidationResult(validator_name="AGENT_EVAL")
+ result.add_success("agent_eval", "ok")
+ return result
+
+ monkeypatch.setattr(cli_module, "run_validation", _tier1)
+ monkeypatch.setattr(cli_module, "_run_agent_eval_or_skip", _tier3)
+
+ result = CliRunner().invoke(
+ cli,
+ [
+ "validate",
+ str(FIXTURE),
+ "--no-llm",
+ "--no-tier2",
+ "--tier3",
+ "--checks",
+ "schema",
+ "--env-mode",
+ "ec2",
+ "--environment-kwarg",
+ "region=us-west-2",
+ "--ek",
+ "launch_mode=attach",
+ "--ek",
+ "instance_id=i-123",
+ ],
+ )
+
+ assert result.exit_code == 0, result.output
+ assert captured["environment_kwarg"] == (
+ "region=us-west-2",
+ "launch_mode=attach",
+ "instance_id=i-123",
+ )
+
+
+def test_validate_helper_routes_environment_kwargs_to_shared_service(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skillevaluator import cli as cli_module
+
+ captured: dict[str, object] = {}
+
+ def _fake_evaluate(self, options: EvaluationOptions, *, progress_reporter=None) -> dict[str, object]:
+ captured["options"] = options
+ return {}
+
+ monkeypatch.setattr(EvaluationService, "evaluate", _fake_evaluate, raising=True)
+
+ cli_module._run_agent_eval_or_skip(
+ FIXTURE,
+ agents="codex",
+ env_mode="ec2",
+ environment_kwarg=(
+ "region=us-west-2",
+ "launch_mode=attach",
+ "instance_id=i-123",
+ ),
+ skip_baseline=False,
+ n_concurrent=None,
+ max_agents=None,
+ validate_source=False,
+ )
+
+ options = captured["options"]
+ assert isinstance(options, EvaluationOptions)
+ assert options.environment_kwarg == (
+ "region=us-west-2",
+ "launch_mode=attach",
+ "instance_id=i-123",
+ )
+
+
def _autopilot_skill(tmp_path: Path) -> Path:
skill = tmp_path / "simple"
shutil.copytree(FIXTURE, skill)
diff --git a/tests/test_harbor_case_id_safety.py b/tests/test_harbor_case_id_safety.py
index eb352f1f..b2bcc4cd 100644
--- a/tests/test_harbor_case_id_safety.py
+++ b/tests/test_harbor_case_id_safety.py
@@ -202,6 +202,7 @@ def test_all_dataset_formats_reject_unsafe_ids_before_creating_output(
def test_valid_case_id_boundaries_round_trip_through_real_harbor_model(tmp_path: Path) -> None:
+ from harbor.models.task.config import TaskConfig
from harbor.models.task.task import Task
case_ids: list[object] = [0, "a", "A_b-c.1", "x" * 128]
@@ -213,6 +214,9 @@ def test_valid_case_id_boundaries_round_trip_through_real_harbor_model(tmp_path:
assert [path.name for path in task_paths] == expected_ids
for expected_id, task_path in zip(expected_ids, task_paths, strict=True):
+ task_config_text = (task_path / "task.toml").read_text(encoding="utf-8")
+ parsed_config = TaskConfig.model_validate_toml(task_config_text)
+ assert parsed_config.schema_version == "1.3"
task = Task(task_path)
assert task.config.metadata["entry_id"] == expected_id
entry = json.loads((task_path / "tests" / "entry.json").read_text(encoding="utf-8"))
@@ -222,6 +226,36 @@ def test_valid_case_id_boundaries_round_trip_through_real_harbor_model(tmp_path:
assert [task["name"] for task in dataset["tasks"]] == [f"nvidia/{case_id}" for case_id in sorted(expected_ids)]
+def test_native_harbor_14_task_stages_without_rewriting_its_schema(tmp_path: Path) -> None:
+ from harbor.models.task.config import TaskConfig
+
+ skill_path = _write_skill(tmp_path, ["case-001"])
+ native_task = skill_path / "evals" / "harbor" / "case-001"
+ native_task.mkdir(parents=True)
+ (native_task / "instruction.md").write_text("Run the native Harbor 0.22 task.\n", encoding="utf-8")
+ native_config_text = (
+ 'schema_version = "1.4"\n\n'
+ "[task]\n"
+ 'name = "nvidia/native-case-001"\n\n'
+ "[metadata]\n"
+ 'entry_id = "case-001"\n\n'
+ "[environment]\n"
+ 'network_mode = "public"\n'
+ )
+ (native_task / "task.toml").write_text(native_config_text, encoding="utf-8")
+ source_config = TaskConfig.model_validate_toml(native_config_text)
+ output_dir = tmp_path / "native-output"
+
+ task_paths = stage_native_harbor_tasks(skill_path, output_dir)
+
+ assert source_config.schema_version == "1.4"
+ assert task_paths == [output_dir / "case-001"]
+ assert (native_task / "task.toml").read_text(encoding="utf-8") == native_config_text
+ staged_config_text = (task_paths[0] / "task.toml").read_text(encoding="utf-8")
+ assert TaskConfig.model_validate_toml(staged_config_text).schema_version == "1.4"
+ assert 'schema_version = "1.4"' in staged_config_text
+
+
def test_existing_task_symlink_is_rejected_without_touching_its_target(tmp_path: Path) -> None:
skill_path = _write_skill(tmp_path, ["case-001"])
output_dir = tmp_path / "generated"
diff --git a/tests/test_harbor_collector_multistep_trajectory.py b/tests/test_harbor_collector_multistep_trajectory.py
new file mode 100644
index 00000000..fac85b15
--- /dev/null
+++ b/tests/test_harbor_collector_multistep_trajectory.py
@@ -0,0 +1,2276 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Harbor 0.22 native multi-step trajectory merge regressions."""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+from pathlib import Path
+from typing import Any
+
+import pytest
+from harbor.models.trajectories import Trajectory
+
+from skillevaluator.tier3.harbor import report_data
+from skillevaluator.tier3.harbor.collector import (
+ COLLECTED_REWARD_JSON_MAX_BYTES,
+ _materialize_trajectory_file,
+ _merged_step_trajectory,
+ _redacted_artifact_text,
+ _save_trials,
+ _save_unscored_trials,
+)
+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS
+
+
+def _trajectory(
+ session_id: str,
+ labels: tuple[str, ...],
+ *,
+ prompt_tokens: int,
+ completion_tokens: int,
+ reasoning_tokens: int,
+ cost_usd: float,
+) -> dict[str, Any]:
+ return {
+ "schema_version": "ATIF-v1.7",
+ "session_id": session_id,
+ "trajectory_id": f"trajectory-{session_id}",
+ "agent": {"name": "codex", "version": "test"},
+ "steps": [
+ {
+ "step_id": index,
+ "source": "agent",
+ "message": label,
+ "tool_calls": [],
+ }
+ for index, label in enumerate(labels, start=1)
+ ],
+ "final_metrics": {
+ "total_prompt_tokens": prompt_tokens,
+ "total_completion_tokens": completion_tokens,
+ "total_cached_tokens": 0,
+ "total_cost_usd": cost_usd,
+ "extra": {
+ "reasoning_output_tokens": reasoning_tokens,
+ "finish_reason": "stop",
+ },
+ },
+ }
+
+
+def _write_multistep_trial(
+ trial_root: Path,
+ trajectories: tuple[tuple[str, dict[str, Any]], ...],
+ *,
+ resume_trajectory: bool,
+ load_trajectory: str | None = None,
+) -> None:
+ agent_config: dict[str, Any] = {"resume_trajectory": resume_trajectory}
+ if load_trajectory is not None:
+ agent_config["load_trajectory"] = load_trajectory
+ (trial_root / "result.json").parent.mkdir(parents=True, exist_ok=True)
+ (trial_root / "result.json").write_text(
+ json.dumps(
+ {
+ "config": {"agent": agent_config},
+ "step_results": [{"step_name": name} for name, _ in trajectories],
+ }
+ ),
+ encoding="utf-8",
+ )
+ for step_name, trajectory in trajectories:
+ agent_dir = trial_root / "steps" / step_name / "agent"
+ agent_dir.mkdir(parents=True)
+ (agent_dir / "trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+
+
+def test_merge_resumed_multistep_trajectory_appends_only_cumulative_suffix(
+ tmp_path: Path,
+) -> None:
+ first = _trajectory(
+ "session-1",
+ ("loaded", "one"),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.125,
+ )
+ second = _trajectory(
+ "session-1",
+ ("loaded", "one", "two"),
+ prompt_tokens=25,
+ completion_tokens=5,
+ reasoning_tokens=4,
+ cost_usd=0.25,
+ )
+ for copied in second["steps"][:2]:
+ copied["is_copied_context"] = True
+ _write_multistep_trial(
+ tmp_path,
+ (
+ ("prepare", first),
+ ("finish", second),
+ ),
+ resume_trajectory=True,
+ load_trajectory="/seed/trajectory.json",
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert [step["message"] for step in merged["steps"]] == ["loaded", "one", "two"]
+ assert [step["step_id"] for step in merged["steps"]] == [1, 2, 3]
+ assert [step["extra"]["harbor_step_name"] for step in merged["steps"]] == [
+ "prepare",
+ "prepare",
+ "finish",
+ ]
+ assert merged["final_metrics"] == {
+ "total_prompt_tokens": 25,
+ "total_completion_tokens": 5,
+ "total_cached_tokens": 0,
+ "total_cost_usd": 0.25,
+ "total_steps": 3,
+ "extra": {
+ "reasoning_output_tokens": 4,
+ "finish_reason": "stop",
+ "harbor_multi_step": True,
+ },
+ }
+
+
+def test_merge_resumed_multistep_accepts_retained_copied_context_suffix(
+ tmp_path: Path,
+) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one", "two", "three"),
+ prompt_tokens=30,
+ completion_tokens=6,
+ reasoning_tokens=3,
+ cost_usd=0.3,
+ )
+ second = _trajectory(
+ "session-1",
+ ("two", "three", "four"),
+ prompt_tokens=40,
+ completion_tokens=8,
+ reasoning_tokens=4,
+ cost_usd=0.4,
+ )
+ for copied in second["steps"][:2]:
+ copied["is_copied_context"] = True
+ _write_multistep_trial(
+ tmp_path,
+ (("prepare", first), ("finish", second)),
+ resume_trajectory=True,
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert [step["message"] for step in merged["steps"]] == ["one", "two", "three", "four"]
+ assert merged["final_metrics"]["total_prompt_tokens"] == 40
+
+
+def test_merge_resumed_multistep_anchors_duplicate_copied_context_to_terminal_suffix(
+ tmp_path: Path,
+) -> None:
+ first = _trajectory(
+ "session-1",
+ ("x", "a", "b", "a", "b"),
+ prompt_tokens=30,
+ completion_tokens=6,
+ reasoning_tokens=3,
+ cost_usd=0.3,
+ )
+ second = _trajectory(
+ "session-1",
+ ("a", "b", "c"),
+ prompt_tokens=40,
+ completion_tokens=8,
+ reasoning_tokens=4,
+ cost_usd=0.4,
+ )
+ for copied in second["steps"][:2]:
+ copied["is_copied_context"] = True
+ _write_multistep_trial(tmp_path, (("prepare", first), ("finish", second)), resume_trajectory=True)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert [step["message"] for step in merged["steps"]] == ["x", "a", "b", "a", "b", "c"]
+
+
+@pytest.mark.parametrize(
+ ("previous_labels", "current_labels"),
+ [
+ (("a", "b", "x", "a"), ("a", "b", "c")),
+ (("a", "b", "c"), ("c", "b", "d")),
+ (("a", "b"), ("a", "b")),
+ ],
+ ids=("earlier-only-match", "reordered-suffix", "all-copied-no-new-step"),
+)
+def test_merge_resumed_multistep_rejects_ambiguous_or_empty_copied_context(
+ tmp_path: Path,
+ previous_labels: tuple[str, ...],
+ current_labels: tuple[str, ...],
+) -> None:
+ first = _trajectory(
+ "session-1",
+ previous_labels,
+ prompt_tokens=30,
+ completion_tokens=6,
+ reasoning_tokens=3,
+ cost_usd=0.3,
+ )
+ second = _trajectory(
+ "session-1",
+ current_labels,
+ prompt_tokens=40,
+ completion_tokens=8,
+ reasoning_tokens=4,
+ cost_usd=0.4,
+ )
+ copied_count = 2
+ for copied in second["steps"][:copied_count]:
+ copied["is_copied_context"] = True
+ _write_multistep_trial(tmp_path, (("prepare", first), ("finish", second)), resume_trajectory=True)
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_merge_copilot_placeholder_session_does_not_deduplicate_independent_fragments(
+ tmp_path: Path,
+) -> None:
+ first = _trajectory(
+ "copilot-cli",
+ ("same output",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "copilot-cli",
+ ("same output",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ first["agent"]["name"] = "copilot"
+ second["agent"]["name"] = "copilot"
+ _write_multistep_trial(
+ tmp_path,
+ (("one", first), ("two", second)),
+ resume_trajectory=True,
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert [step["message"] for step in merged["steps"]] == ["same output", "same output"]
+ assert merged["final_metrics"]["total_prompt_tokens"] == 20
+
+
+def test_merge_unmarked_unique_session_strict_prefix_remains_cumulative(tmp_path: Path) -> None:
+ first = _trajectory(
+ "unique-session",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "unique-session",
+ ("one", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=True)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert [step["message"] for step in merged["steps"]] == ["one", "two"]
+ assert merged["final_metrics"]["total_prompt_tokens"] == 20
+
+
+def test_merge_fails_closed_for_mismatched_cross_step_copied_context(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-1",
+ ("different", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ second["steps"][0]["is_copied_context"] = True
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=True)
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_merge_nonresumed_multistep_trajectory_keeps_independent_fragments(
+ tmp_path: Path,
+) -> None:
+ _write_multistep_trial(
+ tmp_path,
+ (
+ (
+ "prepare",
+ _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.125,
+ ),
+ ),
+ (
+ "finish",
+ _trajectory(
+ "session-2",
+ ("two",),
+ prompt_tokens=25,
+ completion_tokens=5,
+ reasoning_tokens=4,
+ cost_usd=0.25,
+ ),
+ ),
+ ),
+ resume_trajectory=False,
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert [step["message"] for step in merged["steps"]] == ["one", "two"]
+ assert merged["final_metrics"]["total_prompt_tokens"] == 35
+ assert merged["final_metrics"]["total_completion_tokens"] == 7
+ assert merged["final_metrics"]["total_cost_usd"] == 0.375
+ assert merged["final_metrics"]["extra"]["reasoning_output_tokens"] == 5
+
+
+def test_merge_resume_flag_sums_fragments_when_session_prefix_is_not_cumulative(
+ tmp_path: Path,
+) -> None:
+ _write_multistep_trial(
+ tmp_path,
+ (
+ (
+ "prepare",
+ _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.125,
+ ),
+ ),
+ (
+ "finish",
+ _trajectory(
+ "session-1",
+ ("different",),
+ prompt_tokens=25,
+ completion_tokens=5,
+ reasoning_tokens=4,
+ cost_usd=0.25,
+ ),
+ ),
+ ),
+ resume_trajectory=True,
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert [step["message"] for step in merged["steps"]] == ["one", "different"]
+ assert merged["final_metrics"]["total_prompt_tokens"] == 35
+ assert merged["final_metrics"]["total_completion_tokens"] == 7
+ assert merged["final_metrics"]["total_cost_usd"] == 0.375
+ assert merged["final_metrics"]["extra"]["reasoning_output_tokens"] == 5
+
+
+def test_merge_resumed_copied_context_prefix_ignores_marker_metric_and_note_changes(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one", "two"),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.125,
+ )
+ first["steps"][0]["metrics"] = {"prompt_tokens": 4, "completion_tokens": 1}
+ second = _trajectory(
+ "session-1",
+ ("one", "two", "three"),
+ prompt_tokens=25,
+ completion_tokens=5,
+ reasoning_tokens=4,
+ cost_usd=0.25,
+ )
+ for copied in second["steps"][:2]:
+ copied["is_copied_context"] = True
+ copied.pop("metrics", None)
+ copied["extra"] = {"note": "Copied context; metrics already recorded"}
+ _write_multistep_trial(
+ tmp_path,
+ (("prepare", first), ("finish", second)),
+ resume_trajectory=True,
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert [step["message"] for step in merged["steps"]] == ["one", "two", "three"]
+ assert merged["final_metrics"]["total_prompt_tokens"] == 25
+
+
+def test_merge_fails_closed_when_result_lists_a_missing_fragment(tmp_path: Path) -> None:
+ _write_multistep_trial(
+ tmp_path,
+ (
+ (
+ "one",
+ _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ ),
+ ),
+ ),
+ resume_trajectory=False,
+ )
+ result = json.loads((tmp_path / "result.json").read_text(encoding="utf-8"))
+ result["step_results"].append({"step_name": "missing"})
+ (tmp_path / "result.json").write_text(json.dumps(result), encoding="utf-8")
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_merge_fails_closed_for_invalid_atif_fragment(tmp_path: Path) -> None:
+ invalid = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ invalid["steps"].append("not-a-step")
+ _write_multistep_trial(tmp_path, (("one", invalid),), resume_trajectory=False)
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_merge_materializes_continuation_and_external_subagent_refs(tmp_path: Path) -> None:
+ root = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["notes"] = "root note"
+ root["extra"] = {"root": True}
+ root["continued_trajectory_ref"] = "trajectory.cont-1.json"
+ continuation = _trajectory(
+ "continuation-session-1",
+ ("one", "two"),
+ prompt_tokens=25,
+ completion_tokens=5,
+ reasoning_tokens=4,
+ cost_usd=0.25,
+ )
+ continuation["trajectory_id"] = "continuation-1"
+ continuation["notes"] = "continuation note"
+ continuation["extra"] = {"continuation": True}
+ continuation["steps"][0]["is_copied_context"] = True
+ continuation["steps"][1]["observation"] = {
+ "results": [
+ {
+ "content": "delegated",
+ "subagent_trajectory_ref": [{"trajectory_path": "trajectory.subagent.json"}],
+ }
+ ]
+ }
+ subagent = _trajectory(
+ "subagent-session",
+ ("subagent",),
+ prompt_tokens=3,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ subagent.pop("trajectory_id")
+ _write_multistep_trial(tmp_path, (("only", root),), resume_trajectory=False)
+ agent_dir = tmp_path / "steps" / "only" / "agent"
+ (agent_dir / "trajectory.cont-1.json").write_text(json.dumps(continuation), encoding="utf-8")
+ (agent_dir / "trajectory.subagent.json").write_text(json.dumps(subagent), encoding="utf-8")
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert [step["message"] for step in merged["steps"]] == ["one", "two"]
+ assert merged["final_metrics"]["total_prompt_tokens"] == 25
+ assert "continued_trajectory_ref" not in merged
+ assert "root note" in merged["notes"]
+ assert "continuation note" in merged["notes"]
+ embedded_id = merged["subagent_trajectories"][0]["trajectory_id"]
+ assert embedded_id.startswith("skillevaluator-scoped-subagent-")
+ ref = merged["steps"][1]["observation"]["results"][0]["subagent_trajectory_ref"][0]
+ assert ref["trajectory_id"] == embedded_id
+ assert "trajectory_path" not in ref
+ source = merged["extra"]["harbor_multi_step"]["source_trajectories"][0]
+ assert source["trajectory_id"].startswith("skillevaluator-continuation-")
+
+
+def test_merge_recursively_materializes_refs_inside_embedded_subagent(tmp_path: Path) -> None:
+ root = _trajectory(
+ "root-session",
+ ("delegate",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["steps"][0]["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [{"trajectory_id": "embedded-child"}],
+ }
+ ]
+ }
+ child = _trajectory(
+ "child-session",
+ ("child root",),
+ prompt_tokens=3,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = "embedded-child"
+ child["continued_trajectory_ref"] = "trajectory.child-cont.json"
+ root["subagent_trajectories"] = [child]
+
+ child_continuation = _trajectory(
+ "child-session",
+ ("child root", "nested delegate"),
+ prompt_tokens=6,
+ completion_tokens=2,
+ reasoning_tokens=0,
+ cost_usd=0.02,
+ )
+ child_continuation["trajectory_id"] = "child-continuation"
+ child_continuation["steps"][0]["is_copied_context"] = True
+ child_continuation["steps"][1]["observation"] = {
+ "results": [
+ {
+ "content": "grandchild",
+ "subagent_trajectory_ref": [{"trajectory_path": "trajectory.grandchild.json"}],
+ }
+ ]
+ }
+ grandchild = _trajectory(
+ "grandchild-session",
+ ("grandchild",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.001,
+ )
+ grandchild.pop("trajectory_id")
+
+ _write_multistep_trial(tmp_path, (("only", root),), resume_trajectory=False)
+ agent_dir = tmp_path / "steps" / "only" / "agent"
+ (agent_dir / "trajectory.child-cont.json").write_text(json.dumps(child_continuation), encoding="utf-8")
+ (agent_dir / "trajectory.grandchild.json").write_text(json.dumps(grandchild), encoding="utf-8")
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ embedded_child = merged["subagent_trajectories"][0]
+ assert embedded_child["trajectory_id"].startswith("skillevaluator-scoped-subagent-")
+ assert embedded_child["extra"]["harbor_continuation"]["segment_count"] == 2
+ assert "continued_trajectory_ref" not in embedded_child
+ assert [step["message"] for step in embedded_child["steps"]] == ["child root", "nested delegate"]
+ nested_ref = embedded_child["steps"][1]["observation"]["results"][0]["subagent_trajectory_ref"][0]
+ assert "trajectory_path" not in nested_ref
+ assert nested_ref["trajectory_id"] == embedded_child["subagent_trajectories"][0]["trajectory_id"]
+
+
+def test_merge_fails_closed_for_continuation_cycle(tmp_path: Path) -> None:
+ root = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["continued_trajectory_ref"] = "trajectory.cont-1.json"
+ continuation = _trajectory(
+ "session-1",
+ ("two",),
+ prompt_tokens=25,
+ completion_tokens=5,
+ reasoning_tokens=4,
+ cost_usd=0.25,
+ )
+ continuation["continued_trajectory_ref"] = "trajectory.json"
+ _write_multistep_trial(tmp_path, (("only", root),), resume_trajectory=False)
+ agent_dir = tmp_path / "steps" / "only" / "agent"
+ (agent_dir / "trajectory.cont-1.json").write_text(json.dumps(continuation), encoding="utf-8")
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_materialize_preserves_exact_whitespace_in_continuation_source_name(tmp_path: Path) -> None:
+ root = _trajectory(
+ "session-1",
+ ("root",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["continued_trajectory_ref"] = "continuation.json "
+ exact = _trajectory(
+ "session-1",
+ ("exact continuation",),
+ prompt_tokens=5,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.05,
+ )
+ wrong = _trajectory(
+ "session-1",
+ ("wrong clean-name continuation",),
+ prompt_tokens=500,
+ completion_tokens=100,
+ reasoning_tokens=50,
+ cost_usd=5.0,
+ )
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (tmp_path / "continuation.json ").write_text(json.dumps(exact), encoding="utf-8")
+ (tmp_path / "continuation.json").write_text(json.dumps(wrong), encoding="utf-8")
+
+ materialized, _reference_key = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ messages = [step["message"] for step in materialized["steps"]]
+ assert "exact continuation" in messages
+ assert "wrong clean-name continuation" not in messages
+
+
+@pytest.mark.skipif(os.name == "nt", reason="backslash is a separator on Windows, matching Harbor")
+def test_materialize_preserves_literal_posix_backslash_in_continuation_source_name(tmp_path: Path) -> None:
+ root = _trajectory(
+ "session-1",
+ ("root",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ reference = r"cont\next.json"
+ root["continued_trajectory_ref"] = reference
+ literal = _trajectory(
+ "session-1",
+ ("harbor literal",),
+ prompt_tokens=5,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.05,
+ )
+ rewritten = _trajectory(
+ "session-1",
+ ("collector rewritten",),
+ prompt_tokens=500,
+ completion_tokens=100,
+ reasoning_tokens=50,
+ cost_usd=5.0,
+ )
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (tmp_path / reference).write_text(json.dumps(literal), encoding="utf-8")
+ rewritten_path = tmp_path / "cont" / "next.json"
+ rewritten_path.parent.mkdir()
+ rewritten_path.write_text(json.dumps(rewritten), encoding="utf-8")
+
+ materialized, _reference_key = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ messages = [step["message"] for step in materialized["steps"]]
+ assert "harbor literal" in messages
+ assert "collector rewritten" not in messages
+
+
+def test_merge_trusts_explicit_continuation_copied_context_markers(tmp_path: Path) -> None:
+ root = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["continued_trajectory_ref"] = "trajectory.cont-1.json"
+ continuation = _trajectory(
+ "session-1",
+ ("different", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ continuation["steps"][0]["is_copied_context"] = True
+ _write_multistep_trial(tmp_path, (("only", root),), resume_trajectory=False)
+ (tmp_path / "steps" / "only" / "agent" / "trajectory.cont-1.json").write_text(
+ json.dumps(continuation), encoding="utf-8"
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert [step["message"] for step in merged["steps"]] == ["one", "two"]
+
+
+def test_materialize_flattens_continuation_provenance_and_reconciles_total_steps(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ first["trajectory_id"] = "first"
+ first["continued_trajectory_ref"] = "trajectory.cont-1.json"
+ second = _trajectory(
+ "session-1",
+ ("one", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ second["trajectory_id"] = "second"
+ second["steps"][0]["is_copied_context"] = True
+ second["steps"][1]["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [{"trajectory_path": "trajectory.child.json"}],
+ }
+ ]
+ }
+ second["continued_trajectory_ref"] = "trajectory.cont-2.json"
+ third = _trajectory(
+ "session-1",
+ ("two", "three"),
+ prompt_tokens=30,
+ completion_tokens=6,
+ reasoning_tokens=3,
+ cost_usd=0.3,
+ )
+ third["trajectory_id"] = "third"
+ third["steps"][0]["is_copied_context"] = True
+ third["final_metrics"]["total_steps"] = 2
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child.pop("trajectory_id")
+ for name, trajectory in (
+ ("trajectory.json", first),
+ ("trajectory.cont-1.json", second),
+ ("trajectory.cont-2.json", third),
+ ):
+ (tmp_path / name).write_text(json.dumps(trajectory), encoding="utf-8")
+ (tmp_path / "trajectory.child.json").write_text(json.dumps(child), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ assert [step["message"] for step in materialized["steps"]] == ["one", "two", "three"]
+ assert materialized["final_metrics"]["total_steps"] == 3
+ provenance = materialized["extra"]["harbor_continuation"]
+ assert provenance["segment_count"] == 3
+ assert provenance["source_trajectory_ids"] == ["first", "second", "third"]
+ assert len(materialized["subagent_trajectories"]) == 1
+
+
+def test_materialize_prefers_embedded_subagent_when_ref_also_has_missing_path(tmp_path: Path) -> None:
+ root = _trajectory(
+ "root-session",
+ ("delegate",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = "child"
+ root["subagent_trajectories"] = [child]
+ root["steps"][0]["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [{"trajectory_id": "child", "trajectory_path": "missing-sidecar.json"}],
+ }
+ ]
+ }
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ ref = materialized["steps"][0]["observation"]["results"][0]["subagent_trajectory_ref"][0]
+ assert ref == {"trajectory_id": "child"}
+
+
+@pytest.mark.parametrize("dual_key_first", [True, False])
+def test_materialize_reuses_embedded_alias_for_path_only_missing_sidecar(
+ tmp_path: Path,
+ dual_key_first: bool,
+) -> None:
+ root = _trajectory(
+ "root-session",
+ ("delegate",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = "child"
+ root["subagent_trajectories"] = [child]
+ dual_key = {"trajectory_id": "child", "trajectory_path": "missing-sidecar.json"}
+ path_only = {"trajectory_path": "missing-sidecar.json"}
+ root["steps"][0]["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [dual_key, path_only] if dual_key_first else [path_only, dual_key],
+ }
+ ]
+ }
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ refs = materialized["steps"][0]["observation"]["results"][0]["subagent_trajectory_ref"]
+ assert refs == [{"trajectory_id": "child"}, {"trajectory_id": "child"}]
+ assert [item["trajectory_id"] for item in materialized["subagent_trajectories"]] == ["child"]
+
+
+def test_materialize_rejects_conflicting_ids_for_same_external_subagent_file(tmp_path: Path) -> None:
+ root = _trajectory(
+ "root-session",
+ ("first", "second"),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ for step, trajectory_id in zip(root["steps"], ("child-a", "child-b"), strict=True):
+ step["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [
+ {"trajectory_id": trajectory_id, "trajectory_path": "trajectory.child.json"}
+ ],
+ }
+ ]
+ }
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child.pop("trajectory_id")
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (tmp_path / "trajectory.child.json").write_text(json.dumps(child), encoding="utf-8")
+
+ with pytest.raises(ValueError, match="conflicting trajectory_id aliases"):
+ _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+
+@pytest.mark.parametrize(
+ "refs",
+ [
+ (
+ {"trajectory_id": "child", "trajectory_path": "trajectory.child.json"},
+ {"trajectory_path": "trajectory.child.json"},
+ ),
+ (
+ {"trajectory_path": "trajectory.child.json"},
+ {"trajectory_id": "child", "trajectory_path": "trajectory.child.json"},
+ ),
+ ],
+)
+def test_materialize_reuses_supplied_id_for_same_path_regardless_of_ref_order(
+ tmp_path: Path,
+ refs: tuple[dict[str, str], dict[str, str]],
+) -> None:
+ root = _trajectory(
+ "root-session",
+ ("first", "second"),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ for step, ref in zip(root["steps"], refs, strict=True):
+ step["observation"] = {"results": [{"content": "child", "subagent_trajectory_ref": [ref]}]}
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child.pop("trajectory_id")
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (tmp_path / "trajectory.child.json").write_text(json.dumps(child), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ assert [item["trajectory_id"] for item in materialized["subagent_trajectories"]] == ["child"]
+ assert [
+ step["observation"]["results"][0]["subagent_trajectory_ref"][0]["trajectory_id"]
+ for step in materialized["steps"]
+ ] == ["child", "child"]
+
+
+def test_materialize_mixed_embedded_and_path_refs_share_one_canonical_child(tmp_path: Path) -> None:
+ root = _trajectory(
+ "root-session",
+ ("first", "second"),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = "canonical-child"
+ root["subagent_trajectories"] = [child]
+ refs = (
+ {"trajectory_id": "canonical-child", "trajectory_path": "trajectory.child.json"},
+ {"trajectory_path": "trajectory.child.json"},
+ )
+ for step, ref in zip(root["steps"], refs, strict=True):
+ step["observation"] = {"results": [{"content": "child", "subagent_trajectory_ref": [ref]}]}
+ sidecar = dict(child)
+ sidecar.pop("trajectory_id")
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (tmp_path / "trajectory.child.json").write_text(json.dumps(sidecar), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ assert [item["trajectory_id"] for item in materialized["subagent_trajectories"]] == ["canonical-child"]
+ assert [
+ step["observation"]["results"][0]["subagent_trajectory_ref"][0]["trajectory_id"]
+ for step in materialized["steps"]
+ ] == ["canonical-child", "canonical-child"]
+
+
+def test_materialize_does_not_trust_source_forged_continuation_provenance(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ first["trajectory_id"] = "real-a"
+ first["continued_trajectory_ref"] = "trajectory.cont-1.json"
+ first["extra"] = {
+ "harbor_continuation": {
+ "segment_count": 999,
+ "source_session_ids": ["forged-session"],
+ "source_trajectory_ids": ["forged-id"],
+ "source_root_extra": [{"forged": True}],
+ }
+ }
+ second = _trajectory(
+ "session-1",
+ ("one", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ second["trajectory_id"] = "real-b"
+ second["steps"][0]["is_copied_context"] = True
+ (tmp_path / "trajectory.json").write_text(json.dumps(first), encoding="utf-8")
+ (tmp_path / "trajectory.cont-1.json").write_text(json.dumps(second), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ provenance = materialized["extra"]["harbor_continuation"]
+ assert provenance["segment_count"] == 2
+ assert provenance["source_session_ids"] == ["session-1", "session-1"]
+ assert provenance["source_trajectory_ids"] == ["real-a", "real-b"]
+
+
+@pytest.mark.parametrize("poison", [math.nan, math.inf, -math.inf])
+def test_merge_omits_nonfinite_standard_and_extra_aggregate_metrics(
+ tmp_path: Path,
+ poison: float,
+) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-2",
+ ("two",),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ first["final_metrics"]["total_cost_usd"] = poison
+ first["final_metrics"]["extra"]["reasoning_output_tokens"] = poison
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert "total_cost_usd" not in merged["final_metrics"]
+ assert "reasoning_output_tokens" not in merged["final_metrics"]["extra"]
+ assert "NaN" not in json.dumps(merged)
+ assert "Infinity" not in json.dumps(merged)
+
+
+def test_merge_omits_unrepresentable_integer_aggregate_metrics(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-2",
+ ("two",),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ first["final_metrics"]["total_prompt_tokens"] = 10**400
+ first["final_metrics"]["extra"]["reasoning_output_tokens"] = 10**400
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert "total_prompt_tokens" not in merged["final_metrics"]
+ assert "reasoning_output_tokens" not in merged["final_metrics"]["extra"]
+ json.dumps(merged, allow_nan=False)
+
+
+def test_single_step_redaction_omits_unrepresentable_integer_aggregate_metrics(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["final_metrics"]["total_prompt_tokens"] = 10**400
+ trajectory["final_metrics"]["total_steps"] = 10**400
+ trajectory["final_metrics"]["extra"]["reasoning_output_tokens"] = 10**400
+ trajectory["steps"][0]["llm_call_count"] = 10**400
+ trajectory["steps"][0]["metrics"] = {
+ "prompt_tokens": 10**400,
+ "completion_tokens": 10**400,
+ "cached_tokens": 10**400,
+ "extra": {"reasoning_output_tokens": 10**400},
+ }
+
+ redacted = _redacted_artifact_text(tmp_path / "trajectory.json", json.dumps(trajectory))
+
+ assert redacted is not None
+ persisted = json.loads(redacted)
+ assert "total_prompt_tokens" not in persisted["final_metrics"]
+ assert "total_steps" not in persisted["final_metrics"]
+ assert "reasoning_output_tokens" not in persisted["final_metrics"]["extra"]
+ assert "llm_call_count" not in persisted["steps"][0]
+ step_metrics = persisted["steps"][0]["metrics"]
+ assert "prompt_tokens" not in step_metrics
+ assert "completion_tokens" not in step_metrics
+ assert "cached_tokens" not in step_metrics
+ assert "reasoning_output_tokens" not in step_metrics["extra"]
+ Trajectory.model_validate(persisted)
+
+
+def test_single_step_redaction_drops_credential_shaped_extra_key(tmp_path: Path) -> None:
+ credential = "sk-abcdefghijk"
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["extra"] = {credential: "value", "phase": "test"}
+
+ redacted = _redacted_artifact_text(tmp_path / "trajectory.json", json.dumps(trajectory))
+
+ assert redacted is not None
+ assert credential not in redacted
+ assert json.loads(redacted)["extra"] == {"phase": "test"}
+
+
+@pytest.mark.parametrize("location", ["root", "agent", "step", "final_metrics"])
+def test_single_step_redaction_rejects_browser_unsafe_integer_in_arbitrary_extra(
+ tmp_path: Path,
+ location: str,
+) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ if location == "root":
+ trajectory["extra"] = {"custom_counter": 10**400}
+ elif location == "agent":
+ trajectory["agent"]["extra"] = {"custom_counter": 10**400}
+ elif location == "step":
+ trajectory["steps"][0]["extra"] = {"custom_counter": 10**400}
+ else:
+ trajectory["final_metrics"]["extra"]["custom_counter"] = 10**400
+
+ assert _redacted_artifact_text(tmp_path / "trajectory.json", json.dumps(trajectory)) is None
+
+
+def test_saved_deep_trajectory_surfaces_bounded_omission_to_report_loader(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ nested: dict[str, Any] = {}
+ for _index in range(70):
+ nested = {"next": nested}
+ trajectory["extra"] = nested
+ job_dir = tmp_path / "jobs"
+ agent_dir = job_dir / "case-001" / "agent"
+ agent_dir.mkdir(parents=True)
+ (agent_dir / "trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+ trials_dir = tmp_path / "results" / "opencode" / "with-skill" / "trials"
+ reward = {
+ "entry_id": "case-001",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "_trial_name": "case-001",
+ "_trial_root_name": "case-001",
+ }
+
+ _save_trials(
+ [reward],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ trial_out = trials_dir / "case-001"
+ assert not (trial_out / "trajectory.json").exists()
+ manifest = json.loads((trial_out / "artifact_manifest.json").read_text(encoding="utf-8"))
+ assert any(item.get("name") == "trajectory.json" for item in manifest["skipped"])
+ persisted_reward = json.loads((trial_out / "reward.json").read_text(encoding="utf-8"))
+ assert any("trajectory" in warning.casefold() for warning in persisted_reward["warnings"])
+
+ summary_dir = trials_dir.parent
+ (summary_dir / "summary.json").write_text(
+ json.dumps(
+ {
+ "scores": dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "metrics": list(DEFAULT_METRICS),
+ "num_trials": 1,
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ }
+ ),
+ encoding="utf-8",
+ )
+ loaded = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ assert any("trajectory" in warning.casefold() for warning in loaded["rewards"][0]["warnings"])
+
+
+def test_structural_reward_fallback_bounds_identity_and_model_before_publication(tmp_path: Path) -> None:
+ trials_dir = tmp_path / "results" / "opencode" / "with-skill" / "trials"
+ reward = {
+ "entry_id": "e" * 2_100_000,
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "_trial_name": "case-001",
+ "_trial_root_name": "case-001",
+ }
+
+ _save_trials(
+ [reward],
+ trials_dir,
+ None,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ agent_model="m" * 600_000,
+ )
+
+ reward_path = trials_dir / "case-001" / "reward.json"
+ assert reward_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ diagnostics: list[dict[str, Any]] = []
+ persisted = report_data._load_bounded_json(reward_path, diagnostics, artifact="reward")
+ assert isinstance(persisted, dict)
+ assert persisted["evaluation_status"] == "failed"
+ assert "structural limits" in persisted["evaluation_errors"]["collector"]
+ assert len(persisted["entry_id"]) <= 512
+ assert len(persisted["model"]) <= 512
+ assert diagnostics == []
+
+
+def test_collected_reward_byte_reserve_bounds_model_metadata_and_keeps_scoreable(tmp_path: Path) -> None:
+ trials_dir = tmp_path / "results" / "opencode" / "with-skill" / "trials"
+ reward = {
+ "entry_id": "case-001",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "details": {"padding": ""},
+ "_trial_name": "case-001",
+ "_trial_root_name": "case-001",
+ }
+ initial_size = len(json.dumps(reward, separators=(",", ":"), ensure_ascii=True).encode("utf-8"))
+ reward["details"]["padding"] = "x" * (COLLECTED_REWARD_JSON_MAX_BYTES - initial_size)
+
+ _save_trials(
+ [reward],
+ trials_dir,
+ None,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ agent_model="m" * 300_000,
+ agent_model_source="cli",
+ )
+
+ reward_path = trials_dir / "case-001" / "reward.json"
+ assert reward_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert persisted.get("evaluation_status") != "failed"
+ assert all(persisted[metric] == 1.0 for metric in DEFAULT_METRICS)
+ assert persisted["model"].startswith("m")
+ assert len(persisted["model"]) <= 512
+ assert persisted["model_source"] == "cli"
+
+
+def test_merge_overwrites_source_spoofed_step_provenance(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["steps"][0]["extra"] = {
+ "harbor_step_name": "forged",
+ "harbor_original_step_id": 999,
+ }
+ _write_multistep_trial(tmp_path, (("real", trajectory),), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert merged["steps"][0]["extra"]["harbor_step_name"] == "real"
+ assert merged["steps"][0]["extra"]["harbor_original_step_id"] == 1
+
+
+def test_merge_scopes_same_embedded_id_from_independent_step_parents(tmp_path: Path) -> None:
+ parents: list[tuple[str, dict[str, Any]]] = []
+ for step_name, child_message in (("one", "child one"), ("two", "child two")):
+ parent = _trajectory(
+ f"session-{step_name}",
+ (f"parent {step_name}",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ child = _trajectory(
+ f"child-session-{step_name}",
+ (child_message,),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = "child"
+ parent["subagent_trajectories"] = [child]
+ parent["steps"][0]["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [{"trajectory_id": "child"}],
+ }
+ ]
+ }
+ parents.append((step_name, parent))
+ _write_multistep_trial(tmp_path, tuple(parents), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ embedded_ids = [child["trajectory_id"] for child in merged["subagent_trajectories"]]
+ assert len(embedded_ids) == len(set(embedded_ids)) == 2
+ assert [child["steps"][0]["message"] for child in merged["subagent_trajectories"]] == [
+ "child one",
+ "child two",
+ ]
+ refs = [
+ step["observation"]["results"][0]["subagent_trajectory_ref"][0]["trajectory_id"] for step in merged["steps"]
+ ]
+ assert refs == embedded_ids
+ for source_step, child in zip(("one", "two"), merged["subagent_trajectories"], strict=True):
+ scope = child["extra"]["harbor_parent_scope"]
+ assert scope["original_trajectory_id"] == "child"
+ assert scope["parent_scope"] == f"harbor-step:{0 if source_step == 'one' else 1}"
+
+
+def test_explicit_continuation_scopes_same_embedded_id_from_each_parent(tmp_path: Path) -> None:
+ root = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ continuation = _trajectory(
+ "session-1",
+ ("one", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ root["continued_trajectory_ref"] = "trajectory.cont-1.json"
+ continuation["steps"][0]["is_copied_context"] = True
+ for parent, child_message, step_indexes in (
+ (root, "child one", (0,)),
+ (continuation, "child two", (0, 1)),
+ ):
+ child = _trajectory(
+ f"session-{child_message.replace(' ', '-')}",
+ (child_message,),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = "child"
+ parent["subagent_trajectories"] = [child]
+ for index in step_indexes:
+ parent["steps"][index]["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [{"trajectory_id": "child"}],
+ }
+ ]
+ }
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (tmp_path / "trajectory.cont-1.json").write_text(json.dumps(continuation), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ Trajectory.model_validate(materialized)
+ embedded_ids = [child["trajectory_id"] for child in materialized["subagent_trajectories"]]
+ assert len(embedded_ids) == len(set(embedded_ids)) == 2
+ refs = [
+ step["observation"]["results"][0]["subagent_trajectory_ref"][0]["trajectory_id"]
+ for step in materialized["steps"]
+ ]
+ assert refs == embedded_ids
+
+
+def test_merge_fails_closed_for_escaping_trajectory_reference(tmp_path: Path) -> None:
+ root = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["continued_trajectory_ref"] = "../outside.json"
+ _write_multistep_trial(tmp_path, (("only", root),), resume_trajectory=False)
+ (tmp_path / "steps" / "only" / "outside.json").write_text(json.dumps(root), encoding="utf-8")
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_merge_omits_unknown_independent_metrics_instead_of_reporting_partial_totals(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-2",
+ ("two",),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ second["final_metrics"] = {}
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert set(merged["final_metrics"]) == {"total_steps", "extra"}
+ assert merged["final_metrics"]["extra"] == {"harbor_multi_step": True}
+
+
+def test_merge_omits_metric_when_terminal_cumulative_fragment_is_unknown(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-1",
+ ("one", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ second["final_metrics"] = {}
+ second["steps"][0]["is_copied_context"] = True
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=True)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert set(merged["final_metrics"]) == {"total_steps", "extra"}
+
+
+def test_merge_omits_unknown_aggregate_extra_metric_instead_of_terminal_partial_value(
+ tmp_path: Path,
+) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-2",
+ ("two",),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ del first["final_metrics"]["extra"]["reasoning_output_tokens"]
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ Trajectory.model_validate(merged)
+ assert "reasoning_output_tokens" not in merged["final_metrics"]["extra"]
+ assert merged["final_metrics"]["extra"]["finish_reason"] == "stop"
+
+
+def test_synthetic_multistep_id_changes_when_trajectory_content_changes(tmp_path: Path) -> None:
+ ids: list[str] = []
+ for directory, message in (("first", "alpha"), ("second", "beta")):
+ trial_root = tmp_path / directory
+ trajectory = _trajectory(
+ "same-session",
+ (message,),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ _write_multistep_trial(trial_root, (("only", trajectory),), resume_trajectory=False)
+ merged = _merged_step_trajectory(trial_root)
+ assert merged is not None
+ ids.append(merged["trajectory_id"])
+
+ assert ids[0] != ids[1]
+
+
+def test_synthetic_multistep_id_changes_when_authoritative_step_names_change(tmp_path: Path) -> None:
+ ids: list[str] = []
+ trajectory = _trajectory(
+ "same-session",
+ ("same",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ for directory, step_name in (("first", "one"), ("second", "renamed")):
+ trial_root = tmp_path / directory
+ _write_multistep_trial(trial_root, ((step_name, trajectory),), resume_trajectory=False)
+ merged = _merged_step_trajectory(trial_root)
+ assert merged is not None
+ ids.append(merged["trajectory_id"])
+
+ assert ids[0] != ids[1]
+
+
+def test_synthetic_multistep_id_does_not_oracle_redacted_step_name_values(tmp_path: Path) -> None:
+ ids: list[str] = []
+ trajectory = _trajectory(
+ "same-session",
+ ("same",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ for directory, step_name in (("first", "api_key=one"), ("second", "api_key=two")):
+ trial_root = tmp_path / directory
+ _write_multistep_trial(trial_root, ((step_name, trajectory),), resume_trajectory=False)
+ merged = _merged_step_trajectory(trial_root)
+ assert merged is not None
+ ids.append(merged["trajectory_id"])
+
+ assert ids[0] == ids[1]
+
+
+def test_synthetic_multistep_id_changes_with_resume_merge_semantics(tmp_path: Path) -> None:
+ ids: list[str] = []
+ step_counts: list[int] = []
+ first = _trajectory(
+ "same-session",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "same-session",
+ ("one", "two"),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ second["steps"][0]["is_copied_context"] = True
+ for directory, resume in (("resumed", True), ("independent", False)):
+ trial_root = tmp_path / directory
+ _write_multistep_trial(
+ trial_root,
+ (("one", first), ("two", second)),
+ resume_trajectory=resume,
+ )
+ merged = _merged_step_trajectory(trial_root)
+ assert merged is not None
+ ids.append(merged["trajectory_id"])
+ step_counts.append(len(merged["steps"]))
+
+ assert step_counts == [2, 3]
+ assert ids[0] != ids[1]
+
+
+def test_synthetic_multistep_id_does_not_oracle_redacted_secret_values(tmp_path: Path) -> None:
+ ids: list[str] = []
+ for directory, secret in (("first", "synthetic-secret-one"), ("second", "synthetic-secret-two")):
+ trial_root = tmp_path / directory
+ trajectory = _trajectory(
+ "same-session",
+ ("same",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["steps"][0]["tool_calls"] = [
+ {
+ "tool_call_id": "call-1",
+ "function_name": "request",
+ "arguments": {"api_key": secret},
+ }
+ ]
+ _write_multistep_trial(trial_root, (("only", trajectory),), resume_trajectory=False)
+ merged = _merged_step_trajectory(trial_root)
+ assert merged is not None
+ ids.append(merged["trajectory_id"])
+
+ assert ids[0] == ids[1]
+
+
+def test_synthetic_multistep_id_does_not_oracle_redacted_root_extra_values(tmp_path: Path) -> None:
+ ids: list[str] = []
+ for directory, secret in (("first", "synthetic-secret-one"), ("second", "synthetic-secret-two")):
+ trial_root = tmp_path / directory
+ trajectory = _trajectory(
+ "same-session",
+ ("same",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["extra"] = {"api_key": secret}
+ _write_multistep_trial(trial_root, (("only", trajectory),), resume_trajectory=False)
+ merged = _merged_step_trajectory(trial_root)
+ assert merged is not None
+ ids.append(merged["trajectory_id"])
+
+ assert ids[0] == ids[1]
+
+
+def test_minted_subagent_ids_are_stable_across_redacted_secret_only_path_changes(tmp_path: Path) -> None:
+ ids: list[str] = []
+ for directory, reference in (("first", "api_key=one.json"), ("second", "api_key=two.json")):
+ agent_dir = tmp_path / directory
+ agent_dir.mkdir()
+ root = _trajectory(
+ "root-session",
+ ("delegate",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["steps"][0]["observation"] = {
+ "results": [{"content": "child", "subagent_trajectory_ref": [{"trajectory_path": reference}]}]
+ }
+ child = _trajectory(
+ "child-session",
+ ("same child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child.pop("trajectory_id")
+ (agent_dir / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (agent_dir / reference).write_text(json.dumps(child), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(agent_dir, "trajectory.json")
+ ids.append(materialized["subagent_trajectories"][0]["trajectory_id"])
+
+ assert ids[0] == ids[1]
+
+
+def test_distinct_secret_named_sidecars_receive_distinct_parent_local_ids(tmp_path: Path) -> None:
+ root = _trajectory(
+ "root-session",
+ ("one", "two"),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ references = ("api_key=one.json", "api_key=two.json")
+ child = _trajectory(
+ "child-session",
+ ("same child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child.pop("trajectory_id")
+ for step, reference in zip(root["steps"], references, strict=True):
+ step["observation"] = {
+ "results": [{"content": "child", "subagent_trajectory_ref": [{"trajectory_path": reference}]}]
+ }
+ (tmp_path / reference).write_text(json.dumps(child), encoding="utf-8")
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ embedded_ids = [item["trajectory_id"] for item in materialized["subagent_trajectories"]]
+ assert len(embedded_ids) == len(set(embedded_ids)) == 2
+
+
+def test_embedded_secret_only_id_collisions_are_remapped_without_oracle(tmp_path: Path) -> None:
+ emitted_ids: list[list[str]] = []
+ for directory, source_ids in (
+ ("first", ("api_key=one", "api_key=two")),
+ ("second", ("api_key=alpha", "api_key=beta")),
+ ):
+ agent_dir = tmp_path / directory
+ agent_dir.mkdir()
+ root = _trajectory(
+ "root-session",
+ ("delegate",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ children: list[dict[str, Any]] = []
+ refs: list[dict[str, str]] = []
+ for source_id in source_ids:
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = source_id
+ children.append(child)
+ refs.append({"trajectory_id": source_id})
+ root["subagent_trajectories"] = children
+ root["steps"][0]["observation"] = {"results": [{"content": "children", "subagent_trajectory_ref": refs}]}
+ (agent_dir / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(agent_dir, "trajectory.json")
+ redacted = _redacted_artifact_text(agent_dir / "trajectory.json", json.dumps(materialized))
+
+ assert redacted is not None
+ persisted = json.loads(redacted)
+ Trajectory.model_validate(persisted)
+ ids = [child["trajectory_id"] for child in persisted["subagent_trajectories"]]
+ assert len(ids) == len(set(ids)) == 2
+ resolved_refs = persisted["steps"][0]["observation"]["results"][0]["subagent_trajectory_ref"]
+ assert [ref["trajectory_id"] for ref in resolved_refs] == ids
+ emitted_ids.append(ids)
+
+ assert emitted_ids[0] == emitted_ids[1]
+
+
+def test_external_idless_continuation_chains_receive_distinct_reference_scoped_ids(tmp_path: Path) -> None:
+ root = _trajectory(
+ "root-session",
+ ("one", "two"),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ for step, reference in zip(root["steps"], ("base-a.json", "base-b.json"), strict=True):
+ step["observation"] = {
+ "results": [{"content": "child", "subagent_trajectory_ref": [{"trajectory_path": reference}]}]
+ }
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ for suffix, secret in (("a", "secret-one"), ("b", "secret-two")):
+ base = _trajectory(
+ f"base-{suffix}",
+ ("base",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ continuation = _trajectory(
+ f"continuation-{suffix}",
+ ("next",),
+ prompt_tokens=2,
+ completion_tokens=2,
+ reasoning_tokens=0,
+ cost_usd=0.02,
+ )
+ for trajectory in (base, continuation):
+ trajectory.pop("session_id")
+ trajectory.pop("trajectory_id")
+ base["extra"] = {"api_key": secret}
+ base["continued_trajectory_ref"] = f"cont-{suffix}.json"
+ (tmp_path / f"base-{suffix}.json").write_text(json.dumps(base), encoding="utf-8")
+ (tmp_path / f"cont-{suffix}.json").write_text(json.dumps(continuation), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ embedded_ids = [item["trajectory_id"] for item in materialized["subagent_trajectories"]]
+ assert len(embedded_ids) == len(set(embedded_ids)) == 2
+
+
+def test_embedded_sibling_continuations_receive_distinct_parent_local_ids(tmp_path: Path) -> None:
+ root = _trajectory(
+ "root-session",
+ ("delegate",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ children: list[dict[str, Any]] = []
+ refs: list[dict[str, str]] = []
+ for suffix in ("one", "two"):
+ child = _trajectory(
+ f"child-{suffix}",
+ ("base",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child["trajectory_id"] = f"api_key={suffix}"
+ child["continued_trajectory_ref"] = f"continuation-{suffix}.json"
+ continuation = _trajectory(
+ f"continuation-{suffix}",
+ ("next",),
+ prompt_tokens=2,
+ completion_tokens=2,
+ reasoning_tokens=0,
+ cost_usd=0.02,
+ )
+ continuation["trajectory_id"] = f"continuation-api_key={suffix}"
+ children.append(child)
+ refs.append({"trajectory_id": child["trajectory_id"]})
+ (tmp_path / f"continuation-{suffix}.json").write_text(json.dumps(continuation), encoding="utf-8")
+ root["subagent_trajectories"] = children
+ root["steps"][0]["observation"] = {"results": [{"content": "children", "subagent_trajectory_ref": refs}]}
+ (tmp_path / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(tmp_path, "trajectory.json")
+
+ embedded_ids = [item["trajectory_id"] for item in materialized["subagent_trajectories"]]
+ assert len(embedded_ids) == len(set(embedded_ids)) == 2
+ resolved_refs = materialized["steps"][0]["observation"]["results"][0]["subagent_trajectory_ref"]
+ assert [ref["trajectory_id"] for ref in resolved_refs] == embedded_ids
+
+
+def test_merge_preserves_source_agent_and_custom_metric_extras_in_provenance(tmp_path: Path) -> None:
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-2",
+ ("two",),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ first["agent"]["extra"] = {"phase": "base"}
+ second["agent"]["extra"] = {"phase": "next"}
+ first["final_metrics"]["extra"]["llm_calls"] = 1
+ second["final_metrics"]["extra"]["llm_calls"] = 2
+ _write_multistep_trial(tmp_path, (("one", first), ("two", second)), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert "llm_calls" not in merged["final_metrics"]["extra"]
+ sources = merged["extra"]["harbor_multi_step"]["source_trajectories"]
+ assert [source["agent_extra"] for source in sources] == [{"phase": "base"}, {"phase": "next"}]
+ assert [source["final_metrics_extra"]["llm_calls"] for source in sources] == [1, 2]
+
+
+@pytest.mark.parametrize("location", ["custom_step_metric", "step_extra", "custom_final_metric"])
+def test_merge_fails_closed_for_nonfinite_values_outside_known_aggregate_fields(
+ tmp_path: Path,
+ location: str,
+) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ if location == "custom_step_metric":
+ trajectory["steps"][0]["metrics"] = {"custom_metric": math.nan}
+ elif location == "step_extra":
+ trajectory["steps"][0]["extra"] = {"custom_metric": math.inf}
+ else:
+ trajectory["final_metrics"]["extra"]["llm_calls"] = -math.inf
+ _write_multistep_trial(tmp_path, (("only", trajectory),), resume_trajectory=False)
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_merge_handles_json_escaped_lone_surrogate_without_crashing(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("\ud800",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ _write_multistep_trial(tmp_path, (("only", trajectory),), resume_trajectory=False)
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert "\\ud800" in json.dumps(merged, ensure_ascii=True, allow_nan=False)
+
+
+def test_merge_fails_closed_for_non_utf8_trajectory_reference(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["continued_trajectory_ref"] = "\ud800.json"
+ _write_multistep_trial(tmp_path, (("only", trajectory),), resume_trajectory=False)
+
+ assert _merged_step_trajectory(tmp_path) is None
+
+
+def test_maximum_step_count_retains_independent_subagent_reference_budget(tmp_path: Path) -> None:
+ trajectories: list[tuple[str, dict[str, Any]]] = []
+ for index in range(64):
+ trajectory = _trajectory(
+ f"session-{index}",
+ (f"step-{index}",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ trajectories.append((f"step-{index}", trajectory))
+ first = trajectories[0][1]
+ first["steps"][0]["observation"] = {
+ "results": [
+ {
+ "content": "child",
+ "subagent_trajectory_ref": [{"trajectory_path": "trajectory.child.json"}],
+ }
+ ]
+ }
+ _write_multistep_trial(tmp_path, tuple(trajectories), resume_trajectory=False)
+ child = _trajectory(
+ "child-session",
+ ("child",),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ child.pop("trajectory_id")
+ (tmp_path / "steps" / "step-0" / "agent" / "trajectory.child.json").write_text(
+ json.dumps(child),
+ encoding="utf-8",
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert len(merged["steps"]) == 64
+ assert len(merged["subagent_trajectories"]) == 1
+
+
+def test_native_resume_deduplicates_collector_materialized_explicit_continuation_prefix(
+ tmp_path: Path,
+) -> None:
+ first_root = _trajectory(
+ "segment-a",
+ ("a",),
+ prompt_tokens=10,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.1,
+ )
+ first_continuation = _trajectory(
+ "segment-b",
+ ("b",),
+ prompt_tokens=20,
+ completion_tokens=2,
+ reasoning_tokens=0,
+ cost_usd=0.2,
+ )
+ second_root = _trajectory(
+ "segment-a",
+ ("a",),
+ prompt_tokens=10,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.1,
+ )
+ second_continuation = _trajectory(
+ "segment-c",
+ ("b", "c"),
+ prompt_tokens=30,
+ completion_tokens=3,
+ reasoning_tokens=0,
+ cost_usd=0.3,
+ )
+ first_root["continued_trajectory_ref"] = "trajectory.cont.json"
+ second_root["continued_trajectory_ref"] = "trajectory.cont.json"
+ _write_multistep_trial(
+ tmp_path,
+ (("first", first_root), ("second", second_root)),
+ resume_trajectory=True,
+ )
+ (tmp_path / "steps" / "first" / "agent" / "trajectory.cont.json").write_text(
+ json.dumps(first_continuation),
+ encoding="utf-8",
+ )
+ (tmp_path / "steps" / "second" / "agent" / "trajectory.cont.json").write_text(
+ json.dumps(second_continuation),
+ encoding="utf-8",
+ )
+
+ merged = _merged_step_trajectory(tmp_path)
+
+ assert merged is not None
+ assert [step["message"] for step in merged["steps"]] == ["a", "b", "c"]
+ assert merged["final_metrics"]["total_steps"] == 3
+ assert merged["final_metrics"]["total_prompt_tokens"] == 30
+
+
+def test_minted_external_subagent_id_changes_when_content_changes(tmp_path: Path) -> None:
+ ids: list[str] = []
+ for directory, message in (("first", "alpha"), ("second", "beta")):
+ agent_dir = tmp_path / directory
+ agent_dir.mkdir()
+ root = _trajectory(
+ "root-session",
+ ("delegate",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ root["steps"][0]["observation"] = {
+ "results": [
+ {
+ "content": "delegated",
+ "subagent_trajectory_ref": [{"trajectory_path": "trajectory.subagent.json"}],
+ }
+ ]
+ }
+ subagent = _trajectory(
+ "same-subagent-session",
+ (message,),
+ prompt_tokens=1,
+ completion_tokens=1,
+ reasoning_tokens=0,
+ cost_usd=0.01,
+ )
+ subagent.pop("trajectory_id")
+ (agent_dir / "trajectory.json").write_text(json.dumps(root), encoding="utf-8")
+ (agent_dir / "trajectory.subagent.json").write_text(json.dumps(subagent), encoding="utf-8")
+
+ materialized, _ = _materialize_trajectory_file(agent_dir, "trajectory.json")
+ ids.append(materialized["subagent_trajectories"][0]["trajectory_id"])
+
+ assert ids[0] != ids[1]
+
+
+def test_unscored_multistep_trial_persists_materialized_trajectory(tmp_path: Path) -> None:
+ trial_root = tmp_path / "jobs" / "trial-1"
+ first = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ second = _trajectory(
+ "session-2",
+ ("two",),
+ prompt_tokens=20,
+ completion_tokens=4,
+ reasoning_tokens=2,
+ cost_usd=0.2,
+ )
+ _write_multistep_trial(trial_root, (("one", first), ("two", second)), resume_trajectory=False)
+
+ _save_unscored_trials(
+ [],
+ tmp_path / "results",
+ tmp_path / "jobs",
+ agent="opencode",
+ variant="with",
+ )
+
+ trial_out = tmp_path / "results" / "trial-1"
+ persisted = json.loads((trial_out / "trajectory.json").read_text(encoding="utf-8"))
+ Trajectory.model_validate(persisted)
+ assert [step["message"] for step in persisted["steps"]] == ["one", "two"]
+ failure = json.loads((trial_out / "failure.json").read_text(encoding="utf-8"))
+ assert "trajectory.json" in failure["artifacts"]
+
+
+def test_trajectory_redaction_omits_token_id_arrays_and_remains_atif_valid(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("one",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["steps"][0]["metrics"] = {
+ "prompt_tokens": 10,
+ "completion_tokens": 2,
+ "prompt_token_ids": [101, 102],
+ "completion_token_ids": [201],
+ }
+ Trajectory.model_validate(trajectory)
+
+ redacted = _redacted_artifact_text(tmp_path / "trajectory.json", json.dumps(trajectory))
+
+ assert redacted is not None
+ persisted = json.loads(redacted)
+ Trajectory.model_validate(persisted)
+ metrics = persisted["steps"][0]["metrics"]
+ assert "prompt_token_ids" not in metrics
+ assert "completion_token_ids" not in metrics
+
+
+def test_trajectory_redaction_masks_uri_userinfo_and_remains_atif_valid(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("request https://alice:correct@horse@example.test/path failed",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+
+ redacted = _redacted_artifact_text(tmp_path / "trajectory.json", json.dumps(trajectory))
+
+ assert redacted is not None
+ assert "alice" not in redacted
+ assert "correct@horse" not in redacted
+ assert "@example.test" in redacted
+ Trajectory.model_validate_json(redacted)
+
+
+def test_trajectory_redaction_masks_plural_and_nonnumeric_token_fields(tmp_path: Path) -> None:
+ trajectory = _trajectory(
+ "session-1",
+ ("request",),
+ prompt_tokens=10,
+ completion_tokens=2,
+ reasoning_tokens=1,
+ cost_usd=0.1,
+ )
+ trajectory["steps"][0]["tool_calls"] = [
+ {
+ "tool_call_id": "call-1",
+ "function_name": "request",
+ "arguments": {
+ "tokens": ["correct-horse-battery"],
+ "passwords": ["hunter-two-secret"],
+ "access_tokens": ["access-token-secret"],
+ },
+ }
+ ]
+
+ redacted = _redacted_artifact_text(tmp_path / "trajectory.json", json.dumps(trajectory))
+
+ assert redacted is not None
+ assert "correct-horse-battery" not in redacted
+ assert "hunter-two-secret" not in redacted
+ assert "access-token-secret" not in redacted
+ persisted = json.loads(redacted)
+ arguments = persisted["steps"][0]["tool_calls"][0]["arguments"]
+ assert arguments == {
+ "tokens": "",
+ "passwords": "",
+ "access_tokens": "",
+ }
+ assert persisted["final_metrics"]["total_prompt_tokens"] == 10
+ Trajectory.model_validate(persisted)
diff --git a/tests/test_harbor_collector_runtime_failures.py b/tests/test_harbor_collector_runtime_failures.py
index 0f1a1043..8a84df54 100644
--- a/tests/test_harbor_collector_runtime_failures.py
+++ b/tests/test_harbor_collector_runtime_failures.py
@@ -6,11 +6,515 @@
from __future__ import annotations
import json
+from datetime import UTC, datetime
from pathlib import Path
+from typing import Literal
+from uuid import UUID
+
+import pytest
from skillevaluator.evaluation.tier3_report import render_agent_eval_html_report
+from skillevaluator.tier3.harbor import collector as collector_module
+from skillevaluator.tier3.harbor import report_data
from skillevaluator.tier3.harbor.collector import collect_harbor_results
-from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET
+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS
+
+_HARBOR_022_AGENT_RUNTIME_EXCEPTION_TYPES = (
+ "AgentAuthenticationError",
+ "ApiConnectionClosedError",
+ "ApiInternalServerError",
+ "ApiOverloadedError",
+ "ApiProviderResourceNotFoundError",
+ "ApiRateLimitError",
+ "ApiResponseStalledError",
+ "ApiUsageLimitError",
+ "ContextWindowExceededError",
+ "ModelNotFoundError",
+ "NetworkConnectionError",
+ "OutputTokenExceededError",
+ "UnknownApiError",
+)
+
+
+def _expected_typed_runtime_reason(exception_type: str, message: str) -> str:
+ if exception_type == "OutputTokenExceededError":
+ return "OutputTokenExceededError:"
+ return f"{exception_type}: {message}"
+
+
+def _write_actual_harbor_022_result(
+ job_dir: Path,
+ *,
+ reward: float = 1.0,
+ verifier_mode: Literal["present", "null", "missing"] = "present",
+ exception_type: str | None = None,
+ step_rewards: tuple[float, ...] | None = None,
+ step_exception_type: str | None = None,
+) -> str:
+ """Persist a real Harbor 0.22 JobResult and its TrialResult artifact."""
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trial.result import TrialResult
+
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ agent_context = {
+ "n_input_tokens": 7,
+ "n_cache_tokens": 2,
+ "n_output_tokens": 3,
+ }
+ payload: dict[str, object] = {
+ "id": UUID(int=2),
+ "task_name": "nvidia/skillevaluator-case-001",
+ "trial_name": trial_name,
+ "trial_uri": trial_dir.as_uri(),
+ "task_id": {"path": str(job_dir / "task" / "case-001")},
+ "task_checksum": "harbor-0.22-fixture",
+ "config": {
+ "task": {"path": str(job_dir / "task" / "case-001")},
+ "trial_name": trial_name,
+ },
+ "agent_info": {
+ "name": "opencode",
+ "version": "test",
+ "model_info": {"name": "test-model"},
+ },
+ "agent_result": agent_context,
+ "started_at": now,
+ "finished_at": now,
+ "step_results": None,
+ }
+ if verifier_mode == "present":
+ payload["verifier_result"] = {"rewards": {"overall": reward}}
+ elif verifier_mode == "null":
+ payload["verifier_result"] = {"rewards": None}
+ if step_rewards is not None:
+ payload["agent_result"] = None
+ step_results_payload: list[dict[str, object]] = []
+ for index, step_reward in enumerate(step_rewards, start=1):
+ step_result: dict[str, object] = {
+ "step_name": f"step-{index}",
+ "agent_result": agent_context,
+ "verifier_result": {"rewards": {"overall": step_reward}},
+ }
+ if index == 1 and step_exception_type is not None:
+ step_result["exception_info"] = {
+ "exception_type": step_exception_type,
+ "exception_message": "provider step operation failed",
+ "exception_traceback": "",
+ "occurred_at": now,
+ }
+ step_results_payload.append(step_result)
+ payload["step_results"] = step_results_payload
+ if exception_type is not None:
+ payload["exception_info"] = {
+ "exception_type": exception_type,
+ "exception_message": "provider operation failed",
+ "exception_traceback": "",
+ "occurred_at": now,
+ }
+ trial_result = TrialResult.model_validate(payload)
+ job_result = JobResult(
+ id=UUID(int=1),
+ started_at=now,
+ updated_at=now,
+ finished_at=now,
+ n_total_trials=1,
+ stats=JobStats.from_trial_results([trial_result], n_total_trials=1),
+ trial_results=[trial_result],
+ )
+ (trial_dir / "result.json").write_text(trial_result.model_dump_json(indent=2), encoding="utf-8")
+ (job_dir / "result.json").write_text(job_result.model_dump_json(indent=2), encoding="utf-8")
+ for index, step_reward in enumerate(step_rewards or (), start=1):
+ verifier_dir = trial_dir / "steps" / f"step-{index}" / "verifier"
+ verifier_dir.mkdir(parents=True)
+ (verifier_dir / "reward.json").write_text(
+ json.dumps({"overall": step_reward, "entry_id": "case-001"}),
+ encoding="utf-8",
+ )
+ return trial_name
+
+
+@pytest.mark.parametrize(
+ "exception_type",
+ _HARBOR_022_AGENT_RUNTIME_EXCEPTION_TYPES,
+)
+def test_harbor_022_typed_infrastructure_failure_invalidates_present_reward(
+ tmp_path: Path,
+ exception_type: str,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0, exception_type=exception_type)
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ opencode = results["agents"]["opencode"]
+ assert opencode["num_trials_with"] == 0
+ assert opencode["with_skill"] == {}
+ assert opencode["agent_runtime_failures"]["with_skill"] == [
+ {
+ "trial": trial_name,
+ "reason": _expected_typed_runtime_reason(exception_type, "provider operation failed"),
+ }
+ ]
+
+
+def test_harbor_022_safety_refusal_remains_a_scored_zero_outcome(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=0.0)
+ agent_dir = job_dir / trial_name / "agent"
+ agent_dir.mkdir()
+ (agent_dir / "opencode.txt").write_text(
+ json.dumps(
+ {
+ "type": "error",
+ "error": {
+ "name": "AgentSafetyRefusalError",
+ "message": "the model declined this request on safety grounds",
+ },
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ opencode = results["agents"]["opencode"]
+ assert results["execution_status"] == "succeeded"
+ assert opencode["num_trials_with"] == 1
+ assert opencode["agent_runtime_failures"]["with_skill"] == []
+ persisted_summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ assert persisted_summary["overall_score"] == 0.0
+ persisted_reward = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ assert persisted_reward["overall"] == 0.0
+
+
+def test_harbor_022_safety_refusal_exception_is_not_an_infrastructure_failure(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(
+ job_dir,
+ reward=0.0,
+ exception_type="AgentSafetyRefusalError",
+ )
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ opencode = results["agents"]["opencode"]
+ assert results["execution_status"] == "failed"
+ assert opencode["num_trials_with"] == 0
+ assert opencode["agent_runtime_failures"]["with_skill"] == []
+ assert opencode["trial_failures"]["with_skill"] == [
+ {"trial": trial_name, "reason": "AgentSafetyRefusalError: provider operation failed"}
+ ]
+
+
+def test_actual_harbor_022_single_step_success_serializes_and_scores(tmp_path: Path) -> None:
+ from harbor.models.job.result import JobResult
+ from harbor.models.trial.result import TrialResult
+
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+
+ job_result = JobResult.model_validate_json((job_dir / "result.json").read_text(encoding="utf-8"))
+ trial_result = TrialResult.model_validate_json((job_dir / trial_name / "result.json").read_text(encoding="utf-8"))
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert job_result.stats.n_completed_trials == 1
+ assert job_result.stats.n_errored_trials == 0
+ assert trial_result.step_results is None
+ assert trial_result.verifier_result is not None
+ assert trial_result.verifier_result.rewards == {"overall": 1.0}
+ assert results["execution_status"] == "succeeded"
+ assert results["agents"]["opencode"]["num_trials_with"] == 1
+ summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ assert summary["overall_score"] == 1.0
+
+
+@pytest.mark.parametrize("verifier_mode", ("null", "missing"))
+def test_actual_harbor_022_null_or_missing_reward_is_unscored(
+ tmp_path: Path,
+ verifier_mode: Literal["null", "missing"],
+) -> None:
+ from harbor.models.job.result import JobResult
+ from harbor.models.trial.result import TrialResult
+
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, verifier_mode=verifier_mode)
+
+ job_result = JobResult.model_validate_json((job_dir / "result.json").read_text(encoding="utf-8"))
+ trial_result = TrialResult.model_validate_json((job_dir / trial_name / "result.json").read_text(encoding="utf-8"))
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert job_result.stats.n_completed_trials == 1
+ assert job_result.stats.n_errored_trials == 0
+ assert trial_result.step_results is None
+ if verifier_mode == "null":
+ assert trial_result.verifier_result is not None
+ assert trial_result.verifier_result.rewards is None
+ else:
+ assert trial_result.verifier_result is None
+ assert results["execution_status"] == "failed"
+ assert results["agents"]["opencode"]["num_trials_with"] == 0
+ assert results["agents"]["opencode"]["job_failures"]["with_skill"] == (
+ "Harbor evaluation statistics account for 0/1 completed trials"
+ )
+
+
+@pytest.mark.parametrize(
+ ("exception_type", "job_failure"),
+ (
+ ("RuntimeError", "Harbor job did not complete successfully: 1 errored"),
+ ("CancelledError", "Harbor job did not complete successfully: 1 cancelled"),
+ ),
+)
+def test_actual_harbor_022_error_or_cancelled_job_suppresses_reward(
+ tmp_path: Path,
+ exception_type: str,
+ job_failure: str,
+) -> None:
+ from harbor.models.job.result import JobResult
+
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ _write_actual_harbor_022_result(job_dir, reward=1.0, exception_type=exception_type)
+
+ job_result = JobResult.model_validate_json((job_dir / "result.json").read_text(encoding="utf-8"))
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert job_result.stats.n_errored_trials == 1
+ assert job_result.stats.n_cancelled_trials == (exception_type == "CancelledError")
+ assert results["execution_status"] == "failed"
+ assert results["agents"]["opencode"]["num_trials_with"] == 0
+ assert results["agents"]["opencode"]["job_failures"]["with_skill"] == job_failure
+
+
+def test_malformed_job_diagnostic_is_redacted_and_bounded_before_publication(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ job_dir.mkdir(parents=True)
+ github_token = "ghp_" + ("A" * 36)
+ oversized_eval_name = github_token + "-" + ("x" * (3 * 1024 * 1024)) + "-private-tail"
+ (job_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "n_total_trials": 1,
+ "stats": {
+ "n_completed_trials": 1,
+ "n_errored_trials": 0,
+ "n_running_trials": 0,
+ "n_pending_trials": 0,
+ "n_cancelled_trials": 0,
+ "n_retries": 0,
+ "evals": {oversized_eval_name: "invalid"},
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ job_failure = results["agents"]["opencode"]["job_failures"]["with_skill"]
+ assert results["execution_status"] == "failed"
+ assert len(job_failure) <= 4096
+ assert github_token not in job_failure
+ assert "private-tail" not in job_failure
+ summary_path = tmp_path / "results" / "opencode" / "with-skill" / "summary.json"
+ assert summary_path.stat().st_size <= 2 * 1024 * 1024
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
+ assert summary["job_failure"] == job_failure
+ assert github_token not in summary_path.read_text(encoding="utf-8")
+
+
+def test_actual_harbor_022_multistep_root_reward_is_authoritative(tmp_path: Path) -> None:
+ from harbor.models.trial.result import TrialResult
+
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=0.8, step_rewards=(0.0, 0.2))
+
+ trial_result = TrialResult.model_validate_json((job_dir / trial_name / "result.json").read_text(encoding="utf-8"))
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert trial_result.agent_result is None
+ assert trial_result.step_results is not None
+ assert [step.verifier_result.rewards for step in trial_result.step_results if step.verifier_result] == [
+ {"overall": 0.0},
+ {"overall": 0.2},
+ ]
+ assert trial_result.verifier_result is not None
+ assert trial_result.verifier_result.rewards == {"overall": 0.8}
+ assert results["execution_status"] == "succeeded"
+ assert results["agents"]["opencode"]["num_trials_with"] == 1
+ summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ assert summary["overall_score"] == 0.8
+
+
+@pytest.mark.parametrize(
+ "exception_type",
+ _HARBOR_022_AGENT_RUNTIME_EXCEPTION_TYPES,
+)
+def test_actual_harbor_022_multistep_typed_agent_failure_is_infrastructure_failure(
+ tmp_path: Path,
+ exception_type: str,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(
+ job_dir,
+ reward=1.0,
+ step_rewards=(1.0, 1.0),
+ step_exception_type=exception_type,
+ )
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ opencode = results["agents"]["opencode"]
+ assert results["execution_status"] == "failed"
+ assert opencode["num_trials_with"] == 0
+ assert opencode["agent_runtime_failures"]["with_skill"] == [
+ {
+ "trial": trial_name,
+ "reason": _expected_typed_runtime_reason(exception_type, "provider step operation failed"),
+ }
+ ]
+ assert opencode["trial_failures"]["with_skill"] == []
+
+
+def test_actual_harbor_022_multistep_safety_refusal_stays_out_of_infrastructure_failures(
+ tmp_path: Path,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(
+ job_dir,
+ reward=0.0,
+ step_rewards=(0.0, 0.0),
+ step_exception_type="AgentSafetyRefusalError",
+ )
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ opencode = results["agents"]["opencode"]
+ assert results["execution_status"] == "failed"
+ assert opencode["num_trials_with"] == 0
+ assert opencode["agent_runtime_failures"]["with_skill"] == []
+ assert opencode["trial_failures"]["with_skill"] == [
+ {
+ "trial": trial_name,
+ "reason": (
+ "Required judge evaluation failed: collector: Constituent default reward for step step-1 "
+ "is incomplete, non-finite, or failed; the authoritative aggregate was not scored"
+ ),
+ }
+ ]
def _write_complete_job_result(job_dir: Path, trial_names: list[str]) -> None:
@@ -295,14 +799,517 @@ def test_partial_rewards_stay_suppressed_when_not_every_job_error_maps_to_a_tria
def test_complete_low_score_is_execution_success(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
job_dir = jobs_dir / "demo-opencode-with"
- trial_name = "case-001__attempt"
- trial_dir = job_dir / trial_name
- (trial_dir / "verifier").mkdir(parents=True)
- (trial_dir / "verifier" / "reward.json").write_text(
- json.dumps({"overall": 0.1, "entry_id": "case-001"}),
- encoding="utf-8",
- )
- _write_complete_job_result(job_dir, [trial_name])
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ (trial_dir / "verifier").mkdir(parents=True)
+ (trial_dir / "verifier" / "reward.json").write_text(
+ json.dumps({"overall": 0.1, "entry_id": "case-001"}),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ n_attempts=1,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ condition = results["agents"]["opencode"]["conditions"]["with_skill"]
+ assert condition == {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "execution_error_details_total": 0,
+ "execution_error_details_shown": 0,
+ "execution_error_details_truncated": False,
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "runtime_failure_details_total": 0,
+ "runtime_failure_details_shown": 0,
+ "runtime_failure_details_truncated": False,
+ "reward_failure_details_total": 0,
+ "reward_failure_details_shown": 0,
+ "reward_failure_details_truncated": False,
+ }
+ assert results["execution_status"] == "succeeded"
+ assert "error" not in results
+
+
+def test_aggregate_execution_preserves_hidden_child_error_occurrence_counts() -> None:
+ summaries = [
+ {
+ "execution_status": "failed",
+ "execution_errors": ["shared visible error"],
+ "execution_error_details_total": 300,
+ "execution_error_details_shown": 1,
+ "execution_error_details_truncated": True,
+ "expected_attempts": 2,
+ "scored_attempts": 0,
+ },
+ {
+ "execution_status": "failed",
+ "execution_errors": ["shared visible error"],
+ "execution_error_details_total": 2,
+ "execution_error_details_shown": 1,
+ "execution_error_details_truncated": True,
+ "expected_attempts": 3,
+ "scored_attempts": 1,
+ },
+ ]
+
+ aggregate = collector_module._aggregate_execution(summaries)
+
+ assert aggregate == {
+ "execution_status": "failed",
+ "execution_errors": ["shared visible error"],
+ "execution_error_details_total": 302,
+ "execution_error_details_shown": 1,
+ "execution_error_details_truncated": True,
+ "expected_attempts": 5,
+ "scored_attempts": 1,
+ }
+
+
+def test_incomplete_default_reward_is_unscored_and_reported(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ verifier = job_dir / trial_name / "verifier"
+ verifier.mkdir(parents=True)
+ (verifier / "reward.json").write_text(
+ json.dumps(
+ {
+ "metric_set": DEFAULT_METRIC_SET,
+ "security": 1.0,
+ "entry_id": "case-001",
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+ results_dir = tmp_path / "results"
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=results_dir,
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ opencode = results["agents"]["opencode"]
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ assert opencode["num_trials_with"] == 0
+ assert opencode["with_skill"] == {}
+ assert opencode["trial_failures"]["with_skill"] == [
+ {
+ "trial": trial_name,
+ "reason": "Reward metrics are incomplete or non-finite; trial was not scored",
+ }
+ ]
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ report = render_agent_eval_html_report(
+ skill,
+ results_dir,
+ use_llm_judge=False,
+ ).read_text(encoding="utf-8")
+ assert "Reward metrics are incomplete or non-finite; trial was not scored" in report
+
+
+def test_missing_job_result_fails_execution_and_preserves_error_alias(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_dir = jobs_dir / "demo-opencode-with" / "case-001__attempt" / "verifier"
+ trial_dir.mkdir(parents=True)
+ (trial_dir / "reward.json").write_text(json.dumps({"overall": 1.0}), encoding="utf-8")
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ assert results["error"] == results["execution_errors"]
+ assert "result.json" in results["error"][0]
+ persisted = json.loads((tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text())
+ assert persisted["execution_status"] == "failed"
+ assert persisted["scored_attempts"] == 0
+
+
+def test_native_multistep_rewards_count_as_one_logical_attempt(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ for step in ("prepare", "finish"):
+ verifier = job_dir / trial_name / "steps" / step / "verifier"
+ verifier.mkdir(parents=True)
+ (verifier / "reward.json").write_text(
+ json.dumps({"overall": 0.8, "entry_id": "case-001"}),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert results["execution_status"] == "succeeded"
+ assert results["scored_attempts"] == 1
+ assert results["agents"]["opencode"]["num_trials_with"] == 1
+ persisted = json.loads((tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text())
+ assert persisted["num_trials"] == 1
+
+
+@pytest.mark.parametrize(
+ "step_results",
+ [
+ [],
+ [{"step_name": ""}],
+ [{"step_name": "duplicate"}, {"step_name": "duplicate"}],
+ ],
+)
+def test_malformed_authoritative_step_topology_invalidates_root_reward(
+ tmp_path: Path,
+ step_results: list[dict[str, object]],
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["step_results"] = step_results
+ trial_result_path.write_text(json.dumps(trial_result), encoding="utf-8")
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ condition = results["agents"]["opencode"]["conditions"]["with_skill"]
+ assert results["execution_status"] == "failed"
+ assert condition["scored_attempts"] == 0
+ assert "malformed constituent steps" in " ".join(condition["execution_errors"])
+
+
+@pytest.mark.parametrize("verifier_mode", ["null", "missing"])
+def test_harbor_022_multistep_result_without_root_aggregate_is_not_reconstructed(
+ tmp_path: Path,
+ verifier_mode: Literal["null", "missing"],
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(
+ job_dir,
+ verifier_mode=verifier_mode,
+ step_rewards=(0.0, 1.0),
+ )
+ serialized_trial = json.loads((job_dir / trial_name / "result.json").read_text(encoding="utf-8"))
+ diagnostic = collector_module._reward_from_harbor_result(serialized_trial)
+ assert diagnostic is not None
+ assert diagnostic["evaluation_status"] == "failed"
+ assert "not reconstructed or scored" in diagnostic["evaluation_errors"]["collector"]
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ condition = results["agents"]["opencode"]["conditions"]["with_skill"]
+ assert results["execution_status"] == "failed"
+ assert condition["scored_attempts"] == 0
+ # Harbor 0.22 correctly excludes a trial with no root verifier reward from
+ # its eval statistics, so job validation fails before artifact extraction.
+ assert "statistics account for 0/1 completed trials" in " ".join(condition["execution_errors"])
+
+
+def test_harbor_022_complete_envelope_accepts_final_reward_strategy(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0, step_rewards=(0.2, 1.0))
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ final_reward = dict.fromkeys(DEFAULT_METRICS, 1.0)
+ trial_result["verifier_result"]["rewards"] = final_reward
+ trial_result["step_results"][0]["verifier_result"]["rewards"] = {"accuracy": 0.2}
+ trial_result["step_results"][1]["verifier_result"]["rewards"] = final_reward
+ trial_result_path.write_text(json.dumps(trial_result), encoding="utf-8")
+
+ diagnostic = collector_module._reward_from_harbor_result(trial_result)
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert diagnostic is not None
+ assert diagnostic.get("evaluation_status") != "failed"
+ assert results["execution_status"] == "succeeded"
+ assert results["scored_attempts"] == 1
+
+
+def test_legacy_step_reward_with_unrepresentable_integer_fails_closed() -> None:
+ reward = collector_module._reward_from_harbor_result(
+ {
+ "step_results": [
+ {
+ "verifier_result": {
+ "rewards": {"overall": 10**400},
+ }
+ }
+ ]
+ }
+ )
+
+ assert reward is not None
+ assert reward["evaluation_status"] == "failed"
+ assert "non-finite" in reward["evaluation_errors"]["collector"]
+ assert reward["details"]["harbor_rewards"]["overall"] is None
+
+
+def test_large_valid_step_topology_does_not_invalidate_root_reward(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["step_results"] = [{"step_name": f"step-{index}"} for index in range(65)]
+ trial_result_path.write_text(json.dumps(trial_result), encoding="utf-8")
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert results["execution_status"] == "succeeded"
+ assert results["scored_attempts"] == 1
+
+
+@pytest.mark.parametrize(
+ "invalid_reward",
+ [float("nan"), float("inf"), float("-inf"), 10**400],
+ ids=["nan", "positive-infinity", "negative-infinity", "overflowing-integer"],
+)
+def test_invalid_root_reward_number_fails_closed_and_persists_strict_json(
+ tmp_path: Path,
+ invalid_reward: float | int,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["verifier_result"]["rewards"] = {"overall": invalid_reward}
+ trial_result_path.write_text(json.dumps(trial_result), encoding="utf-8")
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ agent = results["agents"]["opencode"]
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ assert agent["num_trials_with"] == 0
+ [failure] = agent["trial_failures"]["with_skill"]
+ assert "non-finite" in failure["reason"]
+ assert len(failure["reason"]) <= 2048
+
+ reward_path = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json"
+ raw_reward = reward_path.read_text(encoding="utf-8")
+
+ def reject_nonstandard_constant(value: str) -> None:
+ raise AssertionError(f"non-standard JSON number persisted: {value}")
+
+ persisted = json.loads(raw_reward, parse_constant=reject_nonstandard_constant)
+ json.dumps(persisted, allow_nan=False)
+ assert persisted["evaluation_status"] == "failed"
+ assert "non-finite" in persisted["evaluation_errors"]["collector"]
+ assert len(persisted["evaluation_errors"]["collector"]) <= 512
+ assert persisted["details"]["harbor_rewards"]["overall"] is None
+
+
+def test_out_of_range_canonical_scores_fail_closed_and_persist_strict_json(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["verifier_result"]["rewards"] = {
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1e308),
+ }
+ trial_result_path.write_text(json.dumps(trial_result), encoding="utf-8")
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ summary_text = (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+
+ def reject_nonstandard_constant(value: str) -> None:
+ raise AssertionError(f"non-standard JSON number persisted: {value}")
+
+ summary = json.loads(summary_text, parse_constant=reject_nonstandard_constant)
+ json.dumps(summary, allow_nan=False)
+ assert summary["execution_status"] == "failed"
+ assert summary["scored_attempts"] == 0
+ assert summary["overall_score"] is None
+
+
+def test_deeply_nested_root_reward_fails_closed_without_recursion_crash(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["verifier_result"]["rewards"] = {
+ "overall": 1.0,
+ "nested_diagnostic": "__DEEPLY_NESTED_VALUE__",
+ }
+ nested_value = "[" * 1_000 + "0" + "]" * 1_000
+ serialized = json.dumps(trial_result).replace('"__DEEPLY_NESTED_VALUE__"', nested_value)
+ trial_result_path.write_text(serialized, encoding="utf-8")
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ [failure] = results["agents"]["opencode"]["trial_failures"]["with_skill"]
+ assert "structural limits" in failure["reason"]
+ reward_path = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json"
+
+ def reject_nonstandard_constant(value: str) -> None:
+ raise AssertionError(f"non-standard JSON number persisted: {value}")
+
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"), parse_constant=reject_nonstandard_constant)
+ json.dumps(persisted, allow_nan=False)
+ assert persisted["evaluation_status"] == "failed"
+ assert "structural limits" in persisted["evaluation_errors"]["collector"]
+ assert "details" not in persisted
+
+
+@pytest.mark.parametrize("wide_value", [[0] * 60_000, "x" * 2_000_000], ids=["nodes", "bytes"])
+def test_wide_root_reward_fails_closed_before_generating_an_unreadable_report_artifact(
+ tmp_path: Path,
+ wide_value: list[int] | str,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["verifier_result"]["rewards"] = {
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "details": {"wide": wide_value},
+ }
+ trial_result_path.write_text(json.dumps(trial_result, separators=(",", ":")), encoding="utf-8")
+
+ results = collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ reward_path = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json"
+ assert reward_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert persisted["evaluation_status"] == "failed"
+ assert "structural limits" in persisted["evaluation_errors"]["collector"]
+ loaded = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ diagnostics = loaded.get("_report_truncation", {}).get("reasons", [])
+ assert not any(diagnostic.get("code") in {"json_bytes", "json_nodes", "json_depth"} for diagnostic in diagnostics)
+
+
+def test_near_node_limit_reward_fails_before_aggregation_reserves_publication_headroom(
+ tmp_path: Path,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0)
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["verifier_result"]["rewards"] = {
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "details": {"wide": [0] * 49_978},
+ }
+ trial_result_path.write_text(json.dumps(trial_result, separators=(",", ":")), encoding="utf-8")
results = collect_harbor_results(
skill_name="demo",
@@ -310,46 +1317,46 @@ def test_complete_low_score_is_execution_success(tmp_path: Path) -> None:
output_dir=tmp_path / "results",
jobs_dir=jobs_dir,
skip_baseline=True,
- n_attempts=1,
expected_cases=1,
expected_case_ids=["case-001"],
expected_trials=1,
+ agent_models={"opencode": {"model": "m", "source": "cli"}},
)
- condition = results["agents"]["opencode"]["conditions"]["with_skill"]
- assert condition == {
- "execution_status": "succeeded",
- "execution_errors": [],
- "expected_attempts": 1,
- "scored_attempts": 1,
- }
- assert results["execution_status"] == "succeeded"
- assert "error" not in results
-
-
-def test_incomplete_default_reward_is_unscored_and_reported(tmp_path: Path) -> None:
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ [failure] = results["agents"]["opencode"]["trial_failures"]["with_skill"]
+ assert "structural limits" in failure["reason"]
+ reward_path = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json"
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert persisted["evaluation_status"] == "failed"
+ assert "structural limits" in persisted["evaluation_errors"]["collector"]
+
+
+@pytest.mark.parametrize(
+ "invalid_reward",
+ [float("nan"), 10**400],
+ ids=["nan", "overflowing-integer"],
+)
+def test_invalid_physical_reward_number_fails_closed_without_crashing(
+ tmp_path: Path,
+ invalid_reward: float | int,
+) -> None:
jobs_dir = tmp_path / "jobs"
job_dir = jobs_dir / "demo-opencode-with"
trial_name = "case-001__attempt"
- verifier = job_dir / trial_name / "verifier"
- verifier.mkdir(parents=True)
- (verifier / "reward.json").write_text(
- json.dumps(
- {
- "metric_set": DEFAULT_METRIC_SET,
- "security": 1.0,
- "entry_id": "case-001",
- }
- ),
+ verifier_dir = job_dir / trial_name / "verifier"
+ verifier_dir.mkdir(parents=True)
+ (verifier_dir / "reward.json").write_text(
+ json.dumps({"overall": invalid_reward, "entry_id": "case-001"}),
encoding="utf-8",
)
_write_complete_job_result(job_dir, [trial_name])
- results_dir = tmp_path / "results"
results = collect_harbor_results(
skill_name="demo",
agents=["opencode"],
- output_dir=results_dir,
+ output_dir=tmp_path / "results",
jobs_dir=jobs_dir,
skip_baseline=True,
expected_cases=1,
@@ -357,32 +1364,34 @@ def test_incomplete_default_reward_is_unscored_and_reported(tmp_path: Path) -> N
expected_trials=1,
)
- opencode = results["agents"]["opencode"]
+ agent = results["agents"]["opencode"]
assert results["execution_status"] == "failed"
assert results["scored_attempts"] == 0
- assert opencode["num_trials_with"] == 0
- assert opencode["with_skill"] == {}
- assert opencode["trial_failures"]["with_skill"] == [
- {
- "trial": trial_name,
- "reason": "Reward metrics are incomplete or non-finite; trial was not scored",
- }
- ]
- skill = tmp_path / "demo"
- skill.mkdir()
- report = render_agent_eval_html_report(
- skill,
- results_dir,
- use_llm_judge=False,
- ).read_text(encoding="utf-8")
- assert "Reward metrics are incomplete or non-finite; trial was not scored" in report
+ [failure] = agent["trial_failures"]["with_skill"]
+ assert "non-finite" in failure["reason"]
+ persisted = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json").read_text(
+ encoding="utf-8"
+ ),
+ parse_constant=lambda value: (_ for _ in ()).throw(AssertionError(value)),
+ )
+ assert persisted["overall"] is None
+ assert persisted["evaluation_status"] == "failed"
+ assert "non-finite" in persisted["evaluation_errors"]["collector"]
-def test_missing_job_result_fails_execution_and_preserves_error_alias(tmp_path: Path) -> None:
+def test_deeply_nested_physical_reward_fails_closed_without_recursion_crash(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
- trial_dir = jobs_dir / "demo-opencode-with" / "case-001__attempt" / "verifier"
- trial_dir.mkdir(parents=True)
- (trial_dir / "reward.json").write_text(json.dumps({"overall": 1.0}), encoding="utf-8")
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ verifier_dir = job_dir / trial_name / "verifier"
+ verifier_dir.mkdir(parents=True)
+ nested_value = "[" * 1_000 + "0" + "]" * 1_000
+ (verifier_dir / "reward.json").write_text(
+ '{"overall":1.0,"entry_id":"case-001","details":' + nested_value + "}",
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
results = collect_harbor_results(
skill_name="demo",
@@ -395,27 +1404,40 @@ def test_missing_job_result_fails_execution_and_preserves_error_alias(tmp_path:
expected_trials=1,
)
+ agent = results["agents"]["opencode"]
assert results["execution_status"] == "failed"
assert results["scored_attempts"] == 0
- assert results["error"] == results["execution_errors"]
- assert "result.json" in results["error"][0]
- persisted = json.loads((tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text())
- assert persisted["execution_status"] == "failed"
- assert persisted["scored_attempts"] == 0
-
-
-def test_native_multistep_rewards_count_as_one_logical_attempt(tmp_path: Path) -> None:
+ [failure] = agent["trial_failures"]["with_skill"]
+ assert "structural limits" in failure["reason"]
+ persisted = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ assert persisted["evaluation_status"] == "failed"
+ assert "structural limits" in persisted["evaluation_errors"]["collector"]
+ assert "details" not in persisted
+
+
+@pytest.mark.parametrize(
+ "invalid_reward",
+ [float("nan"), 10**400],
+ ids=["nan", "overflowing-integer"],
+)
+def test_invalid_step_reward_number_cannot_bypass_missing_root_fail_closed(
+ tmp_path: Path,
+ invalid_reward: float | int,
+) -> None:
jobs_dir = tmp_path / "jobs"
job_dir = jobs_dir / "demo-opencode-with"
- trial_name = "case-001__attempt"
- for step in ("prepare", "finish"):
- verifier = job_dir / trial_name / "steps" / step / "verifier"
- verifier.mkdir(parents=True)
- (verifier / "reward.json").write_text(
- json.dumps({"overall": 0.8, "entry_id": "case-001"}),
- encoding="utf-8",
- )
- _write_complete_job_result(job_dir, [trial_name])
+ trial_name = _write_actual_harbor_022_result(job_dir, reward=1.0, step_rewards=(1.0,))
+ trial_result_path = job_dir / trial_name / "result.json"
+ trial_result = json.loads(trial_result_path.read_text(encoding="utf-8"))
+ trial_result["verifier_result"] = None
+ trial_result["step_results"][0]["verifier_result"]["rewards"] = {"overall": invalid_reward}
+ trial_result_path.write_text(json.dumps(trial_result), encoding="utf-8")
+ step_reward_path = job_dir / trial_name / "steps" / "step-1" / "verifier" / "reward.json"
+ step_reward_path.unlink()
results = collect_harbor_results(
skill_name="demo",
@@ -428,9 +1450,19 @@ def test_native_multistep_rewards_count_as_one_logical_attempt(tmp_path: Path) -
expected_trials=1,
)
- assert results["execution_status"] == "succeeded"
- assert results["scored_attempts"] == 1
- assert results["agents"]["opencode"]["num_trials_with"] == 2
+ agent = results["agents"]["opencode"]
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ [failure] = agent["trial_failures"]["with_skill"]
+ assert "missing" in failure["reason"]
+ persisted = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json").read_text(
+ encoding="utf-8"
+ ),
+ parse_constant=lambda value: (_ for _ in ()).throw(AssertionError(value)),
+ )
+ assert persisted["evaluation_status"] == "failed"
+ assert "missing" in persisted["evaluation_errors"]["collector"]
def test_unexpected_case_fails_execution_coverage(tmp_path: Path) -> None:
@@ -474,6 +1506,7 @@ def _write_reward(
trial_name: str | None = None,
include_entry_id: bool = True,
result_task_name: str | None = None,
+ result_task_path: str | None = None,
) -> None:
trial = jobs_dir / f"demo-opencode-{variant}" / (trial_name or f"{case_id}_attempt{attempt:03d}")
verifier_dirs = [trial / "steps" / step / "verifier" for step in steps] or [trial / "verifier"]
@@ -491,9 +1524,15 @@ def _write_reward(
for verifier_dir in verifier_dirs:
verifier_dir.mkdir(parents=True, exist_ok=True)
(verifier_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
- if result_task_name is not None:
+ if result_task_name is not None or result_task_path is not None:
+ result: dict[str, object] = {"trial_name": trial.name}
+ if result_task_name is not None:
+ result["task_name"] = result_task_name
+ if result_task_path is not None:
+ result["task_id"] = {"path": result_task_path}
+ result["config"] = {"task": {"path": result_task_path}}
(trial / "result.json").write_text(
- json.dumps({"trial_name": trial.name, "task_name": result_task_name}),
+ json.dumps(result),
encoding="utf-8",
)
@@ -619,6 +1658,267 @@ def test_result_derived_case_ids_exercise_partial_pairing_through_collector(tmp_
assert "mcnemar_exact" not in paired
+def test_legacy_result_identity_prefers_task_name_without_trusted_selector_mapping() -> None:
+ result = {
+ "task_name": "publisher/logical-native-id",
+ "task_id": {"path": "/trusted/staging/native-selector"},
+ "config": {"task": {"path": "/trusted/staging/native-selector"}},
+ }
+
+ assert collector_module._entry_id_from_harbor_result(result) == "logical-native-id"
+
+
+@pytest.mark.parametrize("separator", ["-", "_"])
+def test_attempt_like_legacy_identities_are_not_generated_attempt_suffixes(separator: str) -> None:
+ reward = {
+ "entry_id": f"logical{separator}attempt7",
+ "_trial_root_name": f"selector{separator}attempt2",
+ "_trial_name": f"display{separator}attempt9",
+ }
+
+ assert collector_module._attempt_ordinal(reward) is None
+
+
+def test_legacy_truncated_name_with_unbound_attempt_text_is_not_trusted() -> None:
+ trial_name = f"truncated-selector__attempt9__{'a' * 16}"
+
+ assert collector_module._structural_attempt_ordinal(trial_name, "selector-attempt9") is None
+
+
+def test_trusted_task_selector_mapping_does_not_parse_attempt_like_identities_as_ordinals(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ for variant, score in (("with", 1.0), ("without", 0.0)):
+ _write_reward(
+ jobs_dir,
+ variant=variant,
+ case_id="reward-attempt4",
+ attempt=1,
+ score=score,
+ # Harbor 0.22 trial names contain the task selector plus a random
+ # suffix, not a semantic attempt ordinal.
+ trial_name="selector-attempt2__AbCd123",
+ result_task_name="publisher/display-attempt9",
+ result_task_path=f"/trusted/staging/{variant}/selector-attempt2",
+ )
+ _write_variant_job_results(jobs_dir)
+
+ result = _collect(
+ tmp_path,
+ n_attempts=1,
+ expected_cases=1,
+ expected_case_ids=["logical-attempt7"],
+ case_id_by_task_selector={"selector-attempt2": "logical-attempt7"},
+ )
+
+ assert result["execution_status"] == "succeeded"
+ pass_at_k = result["agents"]["opencode"]["pass_at_k"]
+ assert list(pass_at_k["with_skill"]["cases"]) == ["logical-attempt7"]
+ assert list(pass_at_k["without_skill"]["cases"]) == ["logical-attempt7"]
+ assert pass_at_k["lift"]["paired_comparison"]["pairing_status"] == "complete"
+ for variant in ("with-skill", "without-skill"):
+ [trial] = (tmp_path / "results" / "opencode" / variant / "trials").iterdir()
+ persisted = json.loads((trial / "reward.json").read_text(encoding="utf-8"))
+ assert persisted["entry_id"] == "logical-attempt7"
+ assert not any(key.startswith("_") for key in persisted)
+
+
+def test_trusted_task_selector_mapping_uses_structural_attempt_for_custom_only_stop_on_pass(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ job_dir = jobs_dir / "demo-opencode-with"
+ trial_names: list[str] = []
+ for attempt, score, suffix in ((1, 0.0, "AbCd123"), (2, 1.0, "EfGh456")):
+ trial_name = f"demo-opencode-with-selector-attempt2-attempt{attempt:03d}__selector-attempt2__{suffix}"
+ trial_names.append(trial_name)
+ verifier_dir = job_dir / trial_name / "verifier"
+ verifier_dir.mkdir(parents=True)
+ (verifier_dir / "reward.json").write_text(
+ json.dumps(
+ {
+ "overall": score,
+ "metric_set": "custom_only",
+ "custom_metrics": {"native_quality": {"score": score}},
+ }
+ ),
+ encoding="utf-8",
+ )
+ (job_dir / trial_name / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "publisher/display-attempt9",
+ "task_id": {"path": "/trusted/staging/selector-attempt2"},
+ "config": {"task": {"path": "/trusted/staging/selector-attempt2"}},
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, trial_names)
+
+ result = _collect(
+ tmp_path,
+ skip_baseline=True,
+ n_attempts=3,
+ stop_on_pass=True,
+ expected_cases=1,
+ expected_case_ids=["logical-attempt7"],
+ case_id_by_task_selector={"selector-attempt2": "logical-attempt7"},
+ )
+
+ assert result["execution_status"] == "succeeded"
+ assert result["expected_attempts"] == 2
+ assert result["scored_attempts"] == 2
+ with_skill = result["agents"]["opencode"]["pass_at_k"]["with_skill"]
+ assert list(with_skill["cases"]) == ["logical-attempt7"]
+ case = with_skill["cases"]["logical-attempt7"]
+ assert case["first_pass_attempt"] == 2
+ assert case["attempts_used"] == 2
+ assert case["attempts_skipped"] == 1
+
+
+def test_unknown_trusted_selector_cannot_score_as_grader_authored_logical_case(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ _write_reward(
+ jobs_dir,
+ variant="with",
+ case_id="logical-id",
+ attempt=1,
+ score=1.0,
+ result_task_name="publisher/logical-id",
+ result_task_path="/trusted/staging/unexpected-selector",
+ )
+ _write_variant_job_results(jobs_dir, variants=("with",))
+
+ result = _collect(
+ tmp_path,
+ skip_baseline=True,
+ n_attempts=1,
+ expected_cases=1,
+ expected_case_ids=["logical-id"],
+ case_id_by_task_selector={"expected-selector": "logical-id"},
+ )
+
+ assert result["execution_status"] == "failed"
+ assert result["scored_attempts"] == 0
+ assert result["agents"]["opencode"]["pass_at_k"]["with_skill"] == {}
+ [persisted_reward_path] = (tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json")
+ persisted_reward = json.loads(persisted_reward_path.read_text(encoding="utf-8"))
+ assert persisted_reward["entry_id"] == "unknown"
+ assert persisted_reward["evaluation_status"] == "failed"
+
+
+@pytest.mark.parametrize("trial_result_state", ["missing", "malformed", "oversized"])
+def test_trusted_selector_mapping_rejects_sidecar_reward_without_readable_trial_result(
+ tmp_path: Path,
+ trial_result_state: str,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_name = "expected-selector_attempt001"
+ _write_reward(
+ jobs_dir,
+ variant="with",
+ case_id="logical-id",
+ attempt=1,
+ score=1.0,
+ trial_name=trial_name,
+ )
+ trial_result = jobs_dir / "demo-opencode-with" / trial_name / "result.json"
+ if trial_result_state == "malformed":
+ trial_result.write_text("{not-json", encoding="utf-8")
+ elif trial_result_state == "oversized":
+ trial_result.write_text(
+ "x" * (collector_module.DEFAULT_DIAGNOSTIC_ARTIFACT_MAX_BYTES + 1),
+ encoding="utf-8",
+ )
+ _write_variant_job_results(jobs_dir, variants=("with",))
+
+ result = _collect(
+ tmp_path,
+ skip_baseline=True,
+ n_attempts=1,
+ expected_cases=1,
+ expected_case_ids=["logical-id"],
+ case_id_by_task_selector={"expected-selector": "logical-id"},
+ )
+
+ assert result["execution_status"] == "failed"
+ assert result["scored_attempts"] == 0
+ assert result["agents"]["opencode"]["pass_at_k"]["with_skill"] == {}
+ [persisted_reward_path] = (tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json")
+ persisted_reward = json.loads(persisted_reward_path.read_text(encoding="utf-8"))
+ assert persisted_reward["entry_id"] == "unknown"
+ assert persisted_reward["evaluation_status"] == "failed"
+
+
+def test_task_selector_mapping_rejects_duplicate_logical_case_ids(tmp_path: Path) -> None:
+ with pytest.raises(ValueError, match="unique logical case identities"):
+ _collect(
+ tmp_path,
+ expected_case_ids=["logical-id"],
+ case_id_by_task_selector={"selector-a": "logical-id", "selector-b": "logical-id"},
+ )
+
+
+@pytest.mark.parametrize(
+ ("expected_case_ids", "mapping"),
+ [
+ (["logical-a", "logical-b"], {"Selector": "logical-a", "selector": "logical-b"}),
+ (["Logical", "logical"], {"selector-a": "Logical", "selector-b": "logical"}),
+ (["logical-id", "logical-id"], {"selector": "logical-id"}),
+ ],
+ ids=["selector-collision", "logical-id-collision", "duplicate-expected-id"],
+)
+def test_task_selector_mapping_rejects_cross_platform_identity_ambiguity(
+ tmp_path: Path,
+ expected_case_ids: list[str],
+ mapping: dict[str, str],
+) -> None:
+ with pytest.raises(ValueError, match=r"duplicate|collid|unique"):
+ _collect(
+ tmp_path,
+ expected_case_ids=expected_case_ids,
+ case_id_by_task_selector=mapping,
+ )
+
+
+@pytest.mark.parametrize(
+ "result",
+ [
+ {"task_id": {"path": "/trusted/staging/unexpected-selector"}},
+ {},
+ {
+ "task_id": {"path": "/trusted/staging/expected-selector"},
+ "config": {"task": {"path": "/trusted/staging/conflicting-selector"}},
+ },
+ {
+ "task_id": {"path": "/trusted/first/expected-selector"},
+ "config": {"task": {"path": "/trusted/second/expected-selector"}},
+ },
+ ],
+ ids=[
+ "unknown-selector",
+ "missing-selector",
+ "conflicting-selector-sources",
+ "same-basename-conflicting-paths",
+ ],
+)
+def test_trusted_task_selector_mapping_invalidates_unresolved_authored_identity(
+ result: dict[str, object],
+) -> None:
+ reward: dict[str, object] = {
+ "entry_id": "logical-id",
+ "overall": 1.0,
+ }
+
+ collector_module._apply_harbor_result_case_identity(
+ reward,
+ result,
+ {"expected-selector": "logical-id"},
+ )
+
+ assert reward.get("entry_id") != "logical-id"
+ assert collector_module._reward_identity_is_publishable(reward) is False
+
+
def test_stop_on_pass_records_skipped_attempts_in_pass_summary(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
_write_reward(jobs_dir, variant="with", case_id="case-a", attempt=1, score=0.2)
@@ -849,3 +2149,114 @@ def test_expected_case_normalizes_generated_skillevaluator_task_prefix(tmp_path:
assert results["execution_status"] == "succeeded"
assert results["agents"]["opencode"]["pass_at_k"]["with_skill"]["extra_cases"] == []
+
+
+def test_oversized_reward_identities_fail_without_partial_or_oversized_outputs(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ huge_ids = ["a" * 1_100_000, "b" * 1_100_000]
+ trial_names: list[str] = []
+ for index, huge_id in enumerate(huge_ids, start=1):
+ trial_name = f"case-{index:03d}_attempt001"
+ _write_reward(
+ jobs_dir,
+ variant="with",
+ case_id=huge_id,
+ attempt=1,
+ score=1.0,
+ trial_name=trial_name,
+ )
+ trial_names.append(trial_name)
+ _write_complete_job_result(jobs_dir / "demo-opencode-with", trial_names)
+
+ results = _collect(
+ tmp_path,
+ skip_baseline=True,
+ n_attempts=1,
+ expected_cases=2,
+ expected_case_ids=["case-001", "case-002"],
+ expected_trials=2,
+ )
+ summary_path = tmp_path / "results/opencode/with-skill/summary.json"
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
+
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ assert summary["execution_status"] == "failed"
+ assert summary["scored_attempts"] == 0
+ assert summary_path.stat().st_size < collector_module.GENERATED_JSON_MAX_BYTES
+ encoded_results = json.dumps(results, separators=(",", ":"))
+ encoded_summary = json.dumps(summary, separators=(",", ":"))
+ assert huge_ids[0][:1024] not in encoded_results + encoded_summary
+ assert huge_ids[1][:1024] not in encoded_results + encoded_summary
+
+
+@pytest.mark.parametrize(
+ "credential_id",
+ [
+ "sk-abcdefghijk",
+ "case-sk-abcdefghijk",
+ "case_nvapi-abcdefghijk",
+ "case-ghp_abcdefghijklmnopqrstuvwxyz0123456789",
+ ],
+)
+def test_credential_shaped_reward_identity_is_never_an_aggregate_key(
+ tmp_path: Path,
+ credential_id: str,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ _write_reward(
+ jobs_dir,
+ variant="with",
+ case_id=credential_id,
+ attempt=1,
+ score=1.0,
+ trial_name="case-001_attempt001",
+ )
+ _write_complete_job_result(jobs_dir / "demo-opencode-with", ["case-001_attempt001"])
+
+ results = _collect(
+ tmp_path,
+ skip_baseline=True,
+ n_attempts=1,
+ expected_cases=1,
+ expected_case_ids=["case-001"],
+ expected_trials=1,
+ )
+ generated_json = "".join(path.read_text(encoding="utf-8") for path in (tmp_path / "results").rglob("*.json"))
+
+ assert results["execution_status"] == "failed"
+ assert results["scored_attempts"] == 0
+ assert credential_id not in json.dumps(results, separators=(",", ":"))
+ assert credential_id not in generated_json
+
+
+def test_credential_shaped_trial_name_uses_collision_safe_output_alias(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ credential_trial = "case-sk-abcdefghijk"
+ _write_reward(
+ jobs_dir,
+ variant="with",
+ case_id="case-a",
+ attempt=1,
+ score=1.0,
+ trial_name=credential_trial,
+ )
+ _write_complete_job_result(jobs_dir / "demo-opencode-with", [credential_trial])
+
+ results = _collect(
+ tmp_path,
+ skip_baseline=True,
+ n_attempts=1,
+ expected_cases=1,
+ expected_case_ids=["case-a"],
+ expected_trials=1,
+ )
+ results_root = tmp_path / "results"
+ generated_json = "".join(path.read_text(encoding="utf-8") for path in results_root.rglob("*.json"))
+ trial_dirs = [path.name for path in (results_root / "opencode/with-skill/trials").iterdir()]
+
+ assert results["execution_status"] == "succeeded"
+ assert credential_trial not in json.dumps(results, separators=(",", ":"))
+ assert credential_trial not in generated_json
+ assert credential_trial not in trial_dirs
+ assert trial_dirs == ["skillevaluator-trial-collision-000001"]
diff --git a/tests/test_harbor_local_environment.py b/tests/test_harbor_local_environment.py
index a8feba09..eab2a762 100644
--- a/tests/test_harbor_local_environment.py
+++ b/tests/test_harbor_local_environment.py
@@ -3,11 +3,17 @@
from __future__ import annotations
+import asyncio
import os
import shutil
import stat
from pathlib import Path
+import pytest
+from harbor.models.task.config import EnvironmentConfig
+from harbor.models.trial.paths import TrialPaths
+
+from skillevaluator.tier3.harbor import local_sandbox
from skillevaluator.tier3.harbor.local_environment import SkillEvaluatorLocalEnvironment
@@ -56,3 +62,64 @@ def test_raw_path_rewrite_ignores_url_and_local_path_suffixes(tmp_path: Path) ->
rewritten = environment._rewrite_raw_paths(value)
assert rewritten == f'"{local_tests}{os.sep}api" "https://example.invalid/tests/api" "{local_tests}/api"'
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(os.name == "nt", reason="local subprocess backend requires POSIX")
+@pytest.mark.parametrize("secret_name", ["SANDBOX_TOKEN", "MYSECRET", "SOMEKEY", "AUTHENTICATION"])
+def test_real_local_exec_streams_redacted_stdout_and_stderr(
+ tmp_path: Path,
+ secret_name: str,
+) -> None:
+ try:
+ detected_sandbox = local_sandbox.detect("require")
+ except local_sandbox.SandboxUnavailable as exc:
+ pytest.skip(str(exc))
+
+ environment_dir = tmp_path / "environment"
+ environment_dir.mkdir()
+ runtime_root = tmp_path / "runtime"
+ runtime_root.mkdir()
+ environment = SkillEvaluatorLocalEnvironment(
+ environment_dir=environment_dir,
+ environment_name="real-local-streaming",
+ session_id="real-local-streaming",
+ trial_paths=TrialPaths(tmp_path / "trial"),
+ task_env_config=EnvironmentConfig(),
+ runtime_agent="opencode",
+ runtime_root=str(runtime_root),
+ sandbox_mode="require",
+ allow_net=False,
+ strict_reads=True,
+ )
+ secret = "real-sandbox-stream-secret"
+ callbacks: list[tuple[str, str]] = []
+ command = f"printf 'stdout=%s\\n' \"${secret_name}\"; printf 'stderr=%s\\n' \"${secret_name}\" >&2"
+
+ async def on_output(text: str, stream: str) -> None:
+ callbacks.append((text, stream))
+
+ async def exercise() -> tuple[object, str]:
+ await environment.start()
+ try:
+ with environment.scoped_output_callback(on_output):
+ result = await environment.exec(command, env={secret_name: secret})
+ assert environment._sandbox is not None
+ return result, environment._sandbox.plan.backend
+ finally:
+ await environment.stop(delete=False)
+
+ result, backend = asyncio.run(exercise())
+ callback_stdout = "".join(text for text, stream in callbacks if stream == "stdout")
+ callback_stderr = "".join(text for text, stream in callbacks if stream == "stderr")
+
+ assert backend == detected_sandbox.plan.backend
+ assert backend in {"bubblewrap", "seatbelt"}
+ assert callback_stdout == result.stdout
+ assert callback_stderr == result.stderr
+ assert secret not in callback_stdout
+ assert secret not in callback_stderr
+ assert secret not in (result.stdout or "")
+ assert secret not in (result.stderr or "")
+ assert "stdout=" in callback_stdout
+ assert "stderr=" in callback_stderr
diff --git a/tests/test_harbor_local_mode.py b/tests/test_harbor_local_mode.py
index cdda2329..ce556c0e 100644
--- a/tests/test_harbor_local_mode.py
+++ b/tests/test_harbor_local_mode.py
@@ -7,6 +7,8 @@
import asyncio
import contextlib
+import contextvars
+import errno
import json
import logging
import os
@@ -18,6 +20,8 @@
import tempfile
import threading
import time
+import tomllib
+import traceback
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import urlsplit
@@ -25,15 +29,19 @@
import pytest
from harbor.agents.installed.opencode import OpenCode
from harbor.environments.base import BaseEnvironment
+from harbor.models.task.config import EnvironmentConfig
+from harbor.models.trial.paths import TrialPaths
from skillevaluator.provider_config import ProviderConfig
from skillevaluator.tier3.harbor import (
ENV_MODE_LOCAL,
HARBOR_ENV_MODES,
+ HARBOR_NATIVE_ENV_MODES,
LOCAL_AGENT_IMPORT_PATHS,
LOCAL_ENV_IMPORT_PATH,
local_sandbox,
)
+from skillevaluator.tier3.harbor import stream_redaction as stream_redaction_module
from skillevaluator.tier3.harbor.local_agents import (
NVIDIA_BUILD_AGENT_IMPORT_PATHS,
SkillEvaluatorLocalOpenCode,
@@ -48,10 +56,28 @@
_local_agent_credentials,
build_harbor_run_command,
)
+from skillevaluator.tier3.harbor.secret_redaction import redact_secrets_in_log_line
+from skillevaluator.tier3.harbor.secure_docker_environment import SECURE_DOCKER_ENV_IMPORT_PATH
+from skillevaluator.tier3.harbor.stream_redaction import (
+ CommandOutputLimitError,
+ StreamingLogRedactor,
+ StreamingSecretRedactor,
+ _StreamingKnownPatternRedactor,
+)
_NATIVE_WINDOWS_LOCAL_REASON = "native Windows local mode requires WSL2; these checks exercise the POSIX backend"
+class _NoopScopedExecEnvironment:
+ @contextlib.contextmanager
+ def scoped_exec_env(self, _env: dict[str, str]):
+ yield
+
+
+class _LocalCallbackBaseError(BaseException):
+ pass
+
+
def _local_environment(
tmp_path: Path, *, persistent_env: dict[str, str] | None = None
) -> SkillEvaluatorLocalEnvironment:
@@ -70,7 +96,15 @@ def _local_environment(
environment._inherit_agent_keys = False
environment._strict_reads = False
environment._active_processes = {}
+ environment._active_process_secret_values = {}
+ environment._pending_creations = set()
+ environment._creation_secret_values = {}
+ environment._creation_cleanups = {}
+ environment._creation_cleanup_errors = []
+ environment._stop_requested = False
environment._persistent_env = persistent_env or {}
+ environment._output_callbacks = contextvars.ContextVar("test_local_output_callbacks", default=())
+ environment._exec_env_overlays = contextvars.ContextVar("test_local_exec_env_overlays", default=())
environment._sandbox = local_sandbox.Sandbox(local_sandbox.SandboxPlan("none", "advisory-only", "test"))
environment.trial_paths = type(
"TrialPaths",
@@ -100,27 +134,335 @@ def _local_environment(
return environment
+def _initialized_local_environment(
+ tmp_path: Path,
+ *,
+ persistent_env: dict[str, str] | None = None,
+) -> SkillEvaluatorLocalEnvironment:
+ environment_dir = tmp_path / "environment"
+ environment_dir.mkdir()
+ runtime_root = tmp_path / "runtime"
+ runtime_root.mkdir()
+ return SkillEvaluatorLocalEnvironment(
+ environment_dir=environment_dir,
+ environment_name="local-streaming-test",
+ session_id="local-streaming-test",
+ trial_paths=TrialPaths(tmp_path / "trial"),
+ task_env_config=EnvironmentConfig(),
+ runtime_agent="opencode",
+ runtime_root=str(runtime_root),
+ sandbox_mode="off",
+ allow_net=False,
+ persistent_env=persistent_env,
+ )
+
+
def _provider(name: str, *, api_key: str = "k", base_url: str | None = None) -> ProviderConfig:
return ProviderConfig(provider=name, model="m", api_key=api_key, base_url=base_url, litellm_model="m", region=None)
+def test_streaming_log_redactor_is_chunk_partition_invariant() -> None:
+ multiline_secret = "FIRST-HALF\nSECOND-HALF"
+ shorter_secret = "overlap-secret"
+ longer_secret = "overlap-secret-tail"
+ collision_secret = "redacted"
+ known_secret = "".join(("nvapi-", "Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8")) # noqa: FLY002
+ raw = (
+ f"prefix {multiline_secret} {longer_secret} {shorter_secret} "
+ f"{collision_secret} {known_secret} task-granularity suffix"
+ )
+ secrets = {multiline_secret, shorter_secret, longer_secret, collision_secret}
+
+ baseline_redactor = StreamingLogRedactor(secrets)
+ baseline = baseline_redactor.feed(raw) + baseline_redactor.finish()
+ partitions = (
+ [raw],
+ list(raw),
+ [raw[:1], raw[1:17], raw[17:43], raw[43:]],
+ [raw[: len(raw) // 2], raw[len(raw) // 2 :]],
+ )
+
+ for chunks in partitions:
+ redactor = StreamingLogRedactor(secrets)
+ rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
+ assert rendered == baseline
+
+ assert "task-granularity" in baseline
+ for secret in (*secrets, known_secret):
+ assert secret not in baseline
+
+
+def test_exact_stream_redactor_rejects_missing_compiled_prefix_matcher() -> None:
+ redactor = StreamingSecretRedactor({"invariant-secret"})
+ redactor._first_character_re = None
+
+ with pytest.raises(RuntimeError, match="exact redactor invariant"):
+ redactor.feed("ordinary output")
+
+
+def test_known_pattern_redactor_rejects_unknown_candidate_kind() -> None:
+ redactor = _StreamingKnownPatternRedactor()
+ redactor._candidate_kind = "unexpected"
+ redactor._candidate_prefix = "eyJ"
+ redactor._candidate = ["eyJ"]
+
+ with pytest.raises(RuntimeError, match="known-pattern redactor invariant"):
+ redactor.feed("A")
+
+
+@pytest.mark.parametrize("missing_stream", ("stdin", "stdout", "stderr"))
+def test_local_stream_collector_rejects_missing_required_pipe(missing_stream: str) -> None:
+ streams = {"stdin": object(), "stdout": object(), "stderr": object()}
+ streams[missing_stream] = None
+ process = SimpleNamespace(**streams)
+
+ with pytest.raises(RuntimeError, match="local subprocess pipe invariant"):
+ asyncio.run(
+ SkillEvaluatorLocalEnvironment._collect_streamed_output(
+ process, # type: ignore[arg-type]
+ b"",
+ None, # type: ignore[arg-type]
+ )
+ )
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("with_callback", [False, True])
+def test_local_output_limit_fails_closed_and_reaps_process_group(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ with_callback: bool,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+ child_pid_path = environment._workspace / "output-limit-child-pid"
+ overflow_value = "synthetic-output-overflow-secret"
+ callback_chunks: list[str] = []
+ processes: list[asyncio.subprocess.Process] = []
+ create_subprocess_exec = asyncio.create_subprocess_exec
+
+ async def capture_process(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await create_subprocess_exec(*args, **kwargs)
+ processes.append(process)
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ command = (
+ "(sleep 30) & child=$!; "
+ "printf '%s' \"$child\" > output-limit-child-pid; "
+ "printf 12345; printf 67890 >&2; printf '%s' \"$OUTPUT_SECRET\"; wait"
+ )
+
+ async def exercise() -> None:
+ await environment.start()
+ callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
+ with callback_scope:
+ await environment.exec(
+ command,
+ env={"OUTPUT_SECRET": overflow_value},
+ # This case exercises output containment, not timeout racing.
+ # Leave enough startup time for loaded CI hosts to emit the
+ # deliberately over-budget bytes before the sleep descendant.
+ timeout_sec=2.0,
+ )
+
+ monkeypatch.setattr(stream_redaction_module, "MAX_COMMAND_OUTPUT_BYTES", 8, raising=False)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", capture_process)
+
+ with pytest.raises(CommandOutputLimitError, match=r"Command output exceeded the 8-byte safety limit") as caught:
+ asyncio.run(exercise())
+
+ assert len(processes) == 1
+ assert processes[0].returncode is not None
+ assert child_pid_path.is_file()
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ with pytest.raises(ProcessLookupError):
+ os.kill(child_pid, 0)
+ assert len("".join(callback_chunks).encode()) <= 8
+ assert overflow_value not in "".join(callback_chunks)
+ assert overflow_value not in str(caught.value)
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_output_limit_redacts_detached_proxy_components_from_callback(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ diagnostic = "proxy rejected local-limit-user with local-limit-password\n"
+ callback_chunks: list[str] = []
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ async def exercise() -> None:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ await environment.exec(
+ f"printf %s {shlex.quote(diagnostic)}; sleep 0.05; printf overflow",
+ env={
+ "HTTPS_PROXY": "https://local-limit-user:local-limit-password@proxy.invalid:8443",
+ },
+ )
+
+ monkeypatch.setattr(
+ stream_redaction_module,
+ "MAX_COMMAND_OUTPUT_BYTES",
+ len(diagnostic.encode()),
+ raising=False,
+ )
+
+ with pytest.raises(CommandOutputLimitError) as caught:
+ asyncio.run(exercise())
+
+ rendered = "".join(callback_chunks) + str(caught.value)
+ assert "proxy rejected" in "".join(callback_chunks)
+ assert "local-limit-user" not in rendered
+ assert "local-limit-password" not in rendered
+
+
+def test_streaming_log_redactor_preserves_nested_known_pattern_starts() -> None:
+ jwt_secret = ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20))
+ raw = "ask-" + ("a" * 19 + "A1") + "--" + jwt_secret + "☃"
+ expected = redact_secrets_in_log_line(raw)
+ partitions = (
+ [raw],
+ list(raw),
+ [raw[:5], raw[5:26], raw[26:51], raw[51:]],
+ )
+
+ for chunks in partitions:
+ redactor = StreamingLogRedactor(())
+ rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
+ assert rendered == expected
+
+ assert jwt_secret not in expected
+
+
+@pytest.mark.parametrize(
+ "raw",
+ [
+ " crsr_" + "0" * 16 + "sha256~" + "a" * 10 + "☃",
+ "ordinary trailing xsk-abcdefgh",
+ "ordinary trailing sk-short",
+ "ordinary partial nvapi-",
+ "ordinary partial crsr_0123",
+ "ordinary partial eyJ" + "A" * 20,
+ "_" + ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20)),
+ "é" + ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20)),
+ ],
+)
+def test_streaming_log_redactor_matches_batch_at_adjacent_patterns_and_eof(raw: str) -> None:
+ expected = redact_secrets_in_log_line(raw)
+ for chunks in ([raw], list(raw), [raw[: len(raw) // 2], raw[len(raw) // 2 :]]):
+ redactor = StreamingLogRedactor(())
+ rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
+ assert rendered == expected
+
+
+def test_streaming_log_redactor_emits_a_proven_long_key_before_its_terminator() -> None:
+ redactor = StreamingLogRedactor(())
+
+ emitted = redactor.feed("sk-" + "a" * 1_000_000)
+
+ assert emitted == "sk-"
+ assert redactor.finish() == ""
+
+
+@pytest.mark.parametrize(
+ "prefix",
+ [
+ "eyJ",
+ "eyJ" + "A" * 20 + ".",
+ ],
+)
+def test_streaming_log_redactor_bounds_oversized_partial_jwt_candidates(prefix: str) -> None:
+ redactor = StreamingLogRedactor(())
+ raw = prefix + "A" * (512 * 1024)
+ emitted: list[str] = []
+
+ for offset in range(0, len(raw), 64 * 1024):
+ emitted.append(redactor.feed(raw[offset : offset + 64 * 1024]))
+
+ # Stage-zero/stage-one ambiguity must not retain attacker-sized output
+ # until EOF. Once the conservative bound is reached, emit the marker and
+ # discard the rest of that segment in bounded chunks.
+ assert any(emitted)
+ rendered = "".join(emitted) + redactor.finish()
+ assert rendered == "jwt-"
+ assert len(rendered) < 256
+
+
+@pytest.mark.parametrize(
+ "raw",
+ [
+ "eyJ" + "A" * 256 + "." + "B" * 32 + "." + "C" * 32,
+ "eyJ" + "A" * 20 + "." + "B" * 256 + "." + "C" * 32,
+ ],
+)
+def test_oversized_partial_jwt_redaction_discards_later_segments(raw: str) -> None:
+ redactor = StreamingLogRedactor(())
+
+ rendered = redactor.feed(raw + " suffix") + redactor.finish()
+
+ assert rendered == "jwt- suffix"
+ assert "B" * 32 not in rendered
+ assert "C" * 32 not in rendered
+
+
+@pytest.mark.parametrize(
+ "known_secret",
+ [
+ "sk-abcdefgh",
+ "nvapi-abcdefgh",
+ "crsr_0123456789abcdef",
+ "sha256~abcdefgh",
+ ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20)),
+ ],
+)
+def test_exact_replacement_cannot_create_a_visible_known_secret(known_secret: str) -> None:
+ exact_secret = "SECRET88"
+ raw = exact_secret + known_secret
+ for chunks in ([raw], list(raw), [raw[:7], raw[7:11], raw[11:]]):
+ redactor = StreamingLogRedactor({exact_secret})
+ rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
+ assert exact_secret not in rendered
+ assert known_secret not in rendered
+
+
+def test_known_replacement_cannot_create_a_visible_exact_secret() -> None:
+ exact_secret = "sk-"
+ raw = "sk-Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8"
+ for chunks in ([raw], list(raw), [raw[:5], raw[5:19], raw[19:]]):
+ redactor = StreamingLogRedactor({exact_secret})
+ rendered = "".join(redactor.feed(chunk) for chunk in chunks) + redactor.finish()
+ assert raw not in rendered
+ assert exact_secret not in rendered
+
+
def test_local_is_a_registered_env_mode() -> None:
assert ENV_MODE_LOCAL == "local"
assert "local" in HARBOR_ENV_MODES
-def test_build_command_uses_import_paths_not_env_flag() -> None:
+def test_registered_native_env_modes_match_pinned_harbor_release() -> None:
+ from harbor.models.environment_type import EnvironmentType
+
+ assert frozenset(environment.value for environment in EnvironmentType) == HARBOR_NATIVE_ENV_MODES
+
+
+def test_build_command_uses_unified_flags_for_local_imports() -> None:
cmd = build_harbor_run_command(dataset_path="/tmp/ds", agent="opencode", job_name="j", env_mode="local")
joined = " ".join(cmd)
- assert "--environment-import-path" in cmd
- assert LOCAL_ENV_IMPORT_PATH in cmd
- assert "--agent-import-path" in cmd
- assert LOCAL_AGENT_IMPORT_PATHS["opencode"] in cmd
- # local mode must NOT pass Harbor's --env, and must NOT pass -a: harbor's
- # create_agent_from_config prefers the agent NAME over the import path when
- # both are set, which would run the stock (apt-get bootstrapping) agent.
- assert "--env" not in cmd
+ assert "--agent-import-path" not in cmd
+ assert "--environment-import-path" not in cmd
assert "-a" not in cmd
+ assert cmd.count("--agent") == 1
+ assert cmd[cmd.index("--agent") + 1] == LOCAL_AGENT_IMPORT_PATHS["opencode"]
+ assert cmd.count("--env") == 1
+ assert cmd[cmd.index("--env") + 1] == LOCAL_ENV_IMPORT_PATH
assert "sandbox_mode=require" in joined
assert "allow_net=true" in joined # egress on by default for the live agent
assert "runtime_agent=opencode" in joined
@@ -137,12 +479,16 @@ def test_build_command_wires_strict_read_policy(monkeypatch: pytest.MonkeyPatch)
def test_build_command_docker_mode_uses_secure_import_path() -> None:
cmd = build_harbor_run_command(dataset_path="/tmp/ds", agent="codex", job_name="j", env_mode="docker")
- assert "-a" in cmd and cmd[cmd.index("-a") + 1] == "codex"
- assert "--env" not in cmd
- assert "--environment-import-path" in cmd
+ assert "--agent-import-path" not in cmd
+ assert "--environment-import-path" not in cmd
+ assert "-a" not in cmd
+ assert cmd.count("--agent") == 1
+ assert cmd[cmd.index("--agent") + 1] == "codex"
+ assert cmd.count("--env") == 1
+ assert cmd[cmd.index("--env") + 1] == SECURE_DOCKER_ENV_IMPORT_PATH
-def test_docker_bridge_command_uses_custom_agent_import_without_native_agent_flag() -> None:
+def test_docker_bridge_command_uses_unified_flags_for_custom_agent_and_secure_environment() -> None:
import_path = "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorNvidiaBuildCodex"
cmd = build_harbor_run_command(
@@ -153,11 +499,13 @@ def test_docker_bridge_command_uses_custom_agent_import_without_native_agent_fla
agent_import_path=import_path,
)
- assert "--env" not in cmd
- assert "--environment-import-path" in cmd
- assert "--agent-import-path" in cmd
- assert cmd[cmd.index("--agent-import-path") + 1] == import_path
+ assert "--agent-import-path" not in cmd
+ assert "--environment-import-path" not in cmd
assert "-a" not in cmd
+ assert cmd.count("--agent") == 1
+ assert cmd[cmd.index("--agent") + 1] == import_path
+ assert cmd.count("--env") == 1
+ assert cmd[cmd.index("--env") + 1] == SECURE_DOCKER_ENV_IMPORT_PATH
def test_nvidia_build_bridge_agents_are_not_local_mode_agents() -> None:
@@ -169,6 +517,65 @@ def test_nvidia_build_bridge_agents_are_not_local_mode_agents() -> None:
assert LOCAL_AGENT_IMPORT_PATHS["codex"] != NVIDIA_BUILD_AGENT_IMPORT_PATHS["codex"]
+def test_harbor_unified_specs_import_non_abstract_skill_evaluator_classes(tmp_path: Path) -> None:
+ import inspect
+
+ from harbor.agents.factory import AgentFactory
+ from harbor.cli.utils import resolve_environment_spec
+ from harbor.environments.factory import EnvironmentFactory
+ from harbor.models.task.config import EnvironmentConfig as TaskEnvironmentConfig
+ from harbor.models.trial.config import AgentConfig, EnvironmentConfig
+ from harbor.models.trial.paths import TrialPaths
+
+ agent_specs = {
+ "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocalClaudeCode": ("SkillEvaluatorLocalClaudeCode"),
+ "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocalCodex": "SkillEvaluatorLocalCodex",
+ "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorLocalOpenCode": "SkillEvaluatorLocalOpenCode",
+ }
+ for import_path, expected_class_name in agent_specs.items():
+ agent = AgentFactory.create_agent_from_config(
+ AgentConfig(name=import_path, model_name="openai/gpt-4.1-mini"),
+ logs_dir=tmp_path / "agent-logs",
+ )
+
+ assert agent.__class__.__name__ == expected_class_name
+ assert not inspect.isabstract(agent.__class__)
+
+ environment_dir = tmp_path / "environment"
+ environment_dir.mkdir()
+ (environment_dir / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
+ environment_specs = {
+ LOCAL_ENV_IMPORT_PATH: (
+ "SkillEvaluatorLocalEnvironment",
+ {
+ "runtime_agent": "codex",
+ "runtime_root": str(tmp_path / "runtime"),
+ "sandbox_mode": "off",
+ },
+ ),
+ SECURE_DOCKER_ENV_IMPORT_PATH: ("SkillEvaluatorSecureDockerEnvironment", {}),
+ }
+ for index, (import_path, (expected_class_name, kwargs)) in enumerate(environment_specs.items()):
+ environment_type, resolved_import_path = resolve_environment_spec(import_path)
+ assert environment_type is None
+ assert resolved_import_path == import_path
+ environment = EnvironmentFactory.create_environment_from_config(
+ EnvironmentConfig(
+ type=environment_type,
+ import_path=resolved_import_path,
+ kwargs=kwargs,
+ ),
+ environment_dir=environment_dir,
+ environment_name="unified-import-smoke",
+ session_id=f"unified-import-smoke-{index}",
+ trial_paths=TrialPaths(tmp_path / f"trial-{index}"),
+ task_env_config=TaskEnvironmentConfig(),
+ )
+
+ assert environment.__class__.__name__ == expected_class_name
+ assert not inspect.isabstract(environment.__class__)
+
+
def test_local_claude_uses_managed_permissions_and_trial_temp_dir(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -219,9 +626,7 @@ def test_local_nvidia_build_codex_starts_authenticated_host_bridge_and_cleans_up
agent_class = getattr(local_agents, "SkillEvaluatorLocalNvidiaBuildCodex", None)
assert agent_class is not None
- agent = object.__new__(agent_class)
- agent.model_name = "nvidia/nemotron-3-nano-30b-a3b"
- agent._extra_env = {}
+ agent = agent_class(logs_dir=tmp_path, model_name="nvidia/nemotron-3-nano-30b-a3b")
agent.render_instruction = lambda instruction: instruction
agent._resolve_auth_json_path = lambda: None
agent._build_register_skills_command = lambda: None
@@ -232,7 +637,9 @@ def test_local_nvidia_build_codex_starts_authenticated_host_bridge_and_cleans_up
retained_logs: list[tuple[str, str]] = []
origins: list[str] = []
- class Environment:
+ class Environment(_NoopScopedExecEnvironment):
+ default_user = None
+
async def upload_file(self, source: object, destination: object) -> None:
retained_logs.append((str(destination), Path(source).read_text(encoding="utf-8")))
@@ -264,6 +671,12 @@ async def upstream_run(
) -> None:
_ = (instruction, context)
origins.append(self._bridge_origin())
+ effective_config = self._build_effective_config()
+ await self._upload_effective_config(
+ environment,
+ effective_config,
+ (self._REMOTE_CODEX_HOME / "config.toml").as_posix(),
+ )
await self.exec_as_agent(
environment,
command="codex exec --model nemotron-3-nano-30b-a3b -- test",
@@ -280,18 +693,18 @@ async def upstream_run(
parsed = urlsplit(origins[0])
assert parsed.hostname == "127.0.0.1"
assert parsed.port is not None
- setup_commands = [(command, env) for command, env in calls if "model_provider" in command]
- assert len(setup_commands) == 1
- setup_command, setup_env = setup_commands[0]
- assert f'base_url = "{origins[0]}/v1"' in setup_command
- assert "api.openai.com" not in setup_command
- assert "real-nvidia-key" not in setup_command
- assert setup_env["OPENAI_API_KEY"] not in {"real-nvidia-key", "nvidia-build-loopback"}
+ config_upload = next(content for destination, content in retained_logs if destination.endswith("/config.toml"))
+ config = tomllib.loads(config_upload)
+ assert config["model_provider"] == "openai_compatible"
+ assert config["openai_base_url"] == f"{origins[0]}/v1"
+ assert config["model_providers"]["openai_compatible"]["base_url"] == f"{origins[0]}/v1"
+ assert "api.openai.com" not in config_upload
+ assert "real-nvidia-key" not in config_upload
client_command, client_env = next((command, env) for command, env in calls if "codex exec" in command)
assert "env -u NVIDIA_API_KEY" in client_command
assert "NVIDIA_API_KEY" not in client_env
- assert client_env["OPENAI_API_KEY"] == setup_env["OPENAI_API_KEY"]
- assert retained_logs and retained_logs[0][0].endswith("nvidia-build-bridge.log")
+ assert client_env["OPENAI_API_KEY"] not in {"real-nvidia-key", "nvidia-build-loopback"}
+ assert any(destination.endswith("nvidia-build-bridge.log") for destination, _content in retained_logs)
with socket.socket() as probe:
probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -527,16 +940,21 @@ async def exercise() -> None:
def test_nvidia_build_codex_bridge_isolated_from_client_and_cleans_up(
monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
) -> None:
from harbor.agents.installed.codex import Codex
- agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
- agent.model_name = "nvidia/meta/llama-3.1-8b-instruct"
+ agent = SkillEvaluatorNvidiaBuildCodex(
+ logs_dir=tmp_path,
+ model_name="nvidia/meta/llama-3.1-8b-instruct",
+ )
calls: list[tuple[str, dict[str, str]]] = []
root_calls: list[tuple[str, dict[str, str]]] = []
uploads: list[tuple[str, str, int]] = []
- class Environment:
+ class Environment(_NoopScopedExecEnvironment):
+ default_user = None
+
async def upload_file(self, source: object, destination: object) -> None:
source_path = Path(source)
uploads.append(
@@ -588,7 +1006,7 @@ async def root_exec(
asyncio.run(agent.run("test", Environment(), None))
- assert len(uploads) == 1
+ assert len(uploads) == 2
assert uploads[0][0].endswith("nvidia-build-bridge.py")
secret_handoff = next(
(command, env) for command, env in root_calls if "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_API_KEY" in env
@@ -621,16 +1039,18 @@ async def root_exec(
assert "kill -0" in health_command
assert "/healthz" not in health_command
assert all("NVIDIA_API_KEY" not in env for _, env in [*calls, *root_calls])
- setup_command, setup_env = next((command, env) for command, env in calls if "model_provider" in command)
- assert 'model_provider = "openai_compatible"' in setup_command
- assert "[model_providers.openai_compatible]" in setup_command
- assert 'base_url = "http://127.0.0.1:43123/v1"' in setup_command
- assert 'wire_api = "responses"' in setup_command
- assert "openai_base_url" not in setup_command
+ config_upload = next(content for destination, content, _mode in uploads if destination.endswith("/config.toml"))
+ config = tomllib.loads(config_upload)
+ assert config["model_provider"] == "openai_compatible"
+ assert config["openai_base_url"] == "http://127.0.0.1:43123/v1"
+ assert config["model_providers"]["openai_compatible"] == {
+ "name": "OpenAI-compatible provider",
+ "base_url": "http://127.0.0.1:43123/v1",
+ "env_key": "OPENAI_API_KEY",
+ "wire_api": "responses",
+ }
assert all("openai_base_url" not in command for command, _ in calls)
- assert "nvidia-secret" not in setup_command
- assert setup_env["OPENAI_API_KEY"] == client_token
- assert "OPENAI_BASE_URL" not in setup_env
+ assert "nvidia-secret" not in config_upload
client_command, client_env = next((command, env) for command, env in calls if "codex exec" in command)
assert "NVIDIA_API_KEY" not in client_env
assert "OPENAI_BASE_URL" not in client_env
@@ -924,11 +1344,6 @@ def reject_key_read() -> str:
assert uploads == []
assert executions == []
- agent._nvidia_build_bridge_client_env = {"OPENAI_API_KEY": "bridge-client-token-secret"}
- with pytest.raises(RuntimeError, match="protected sensitive-value transport"):
- asyncio.run(agent.exec_as_agent(UnsafeEnvironment(), command="codex exec -- test"))
- assert executions == []
-
def test_nvidia_build_bridge_health_failure_cleans_up_before_raising(
monkeypatch: pytest.MonkeyPatch,
@@ -1073,140 +1488,51 @@ def test_nvidia_build_bridge_wraps_compound_codex_shell_commands_before_unsettin
agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
agent.model_name = "nvidia/model"
agent._nvidia_build_bridge_origin = "http://127.0.0.1:43123"
- agent._nvidia_build_bridge_client_env = {
- "OPENAI_API_KEY": "bridge-client-token-secret",
- "OPENAI_BASE_URL": "http://127.0.0.1:43123/v1",
- }
- agent._extra_env = {
- "NVIDIA_API_KEY": "extra-upstream-secret",
- "CODEX_HOME": "/logs/agent/codex-home",
- }
- log_records: list[tuple[object, object]] = []
- agent.logger = SimpleNamespace(debug=lambda message, *args, **kwargs: log_records.append((message, (args, kwargs))))
+ agent._nvidia_build_bridge_client_env = {"OPENAI_API_KEY": "bridge-client-token-secret"}
captured: list[tuple[str, dict[str, str]]] = []
+ scoped: list[dict[str, str]] = []
simple_command = "codex exec --model model -- test"
compound_command = 'if [ -d "$CODEX_HOME/sessions" ]; then cp -R "$CODEX_HOME/sessions" /logs/agent/sessions; fi'
class Environment:
- async def exec_with_sensitive_env(
- self,
- command: str,
- env: dict[str, str] | None = None,
- **_kwargs: object,
- ) -> SimpleNamespace:
- captured.append((command, dict(env or {})))
- return SimpleNamespace(return_code=0, stdout="", stderr="")
+ @contextlib.contextmanager
+ def scoped_exec_env(self, values: dict[str, str]):
+ scoped.append(dict(values))
+ yield
- async def reject_logged_exec(
+ async def raw_exec(
_self: Codex,
_environment: object,
command: str,
env: dict[str, str] | None = None,
**_kwargs: object,
) -> SimpleNamespace:
- _ = (command, env)
- raise AssertionError("bridge credentials must bypass Harbor's logging wrapper")
+ captured.append((command, dict(env or {})))
+ return SimpleNamespace(return_code=0, stdout="", stderr="")
- monkeypatch.setattr(Codex, "exec_as_agent", reject_logged_exec)
+ monkeypatch.setattr(Codex, "exec_as_agent", raw_exec)
+ environment = Environment()
for original_command in (simple_command, compound_command):
asyncio.run(
agent.exec_as_agent(
- Environment(),
+ environment,
command=original_command,
- env={"NVIDIA_API_KEY": "per-call-upstream-secret"},
+ env={"NVIDIA_API_KEY": "must-not-leak"},
)
)
+ unset_prefix = "env -u NVIDIA_API_KEY -u OPENAI_BASE_URL -u OPENAI_API_BASE bash -o pipefail -c"
assert [command for command, _ in captured] == [
- f"env -u NVIDIA_API_KEY bash -o pipefail -c {shlex.quote('codex exec --model nvidia/model -- test')}",
- f"env -u NVIDIA_API_KEY bash -o pipefail -c {shlex.quote(compound_command)}",
+ f"{unset_prefix} {shlex.quote('codex exec --model nvidia/model -- test')}",
+ f"{unset_prefix} {shlex.quote(compound_command)}",
]
assert all("NVIDIA_API_KEY" not in env for _, env in captured)
assert all(env["OPENAI_API_KEY"] == "bridge-client-token-secret" for _, env in captured)
- assert all(env["CODEX_HOME"] == "/logs/agent/codex-home" for _, env in captured)
- assert log_records == []
-
-
-def test_nvidia_build_bridge_client_preserves_pipefail_in_real_inner_shell() -> None:
- from harbor.agents.installed.base import NonZeroAgentExitCodeError
-
- agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
- agent.model_name = "nvidia/model"
- agent._extra_env = {"NVIDIA_API_KEY": "upstream-key-must-be-unset"}
- agent._nvidia_build_bridge_origin = "http://127.0.0.1:43123"
- agent._nvidia_build_bridge_client_env = {"OPENAI_API_KEY": "bridge-client-token-secret"}
-
- class Environment:
- async def exec_with_sensitive_env(
- self,
- command: str,
- env: dict[str, str] | None = None,
- **_kwargs: object,
- ) -> SimpleNamespace:
- process = await asyncio.create_subprocess_exec(
- "bash",
- "-c",
- command,
- env={**os.environ, **(env or {})},
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- )
- stdout, stderr = await process.communicate()
- return SimpleNamespace(
- return_code=process.returncode,
- stdout=stdout.decode(),
- stderr=stderr.decode(),
- )
-
- with pytest.raises(NonZeroAgentExitCodeError, match="exit code 1"):
- asyncio.run(agent.exec_as_agent(Environment(), command="false | true"))
-
-
-def test_nvidia_build_bridge_client_failure_does_not_expose_sensitive_output(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- from harbor.agents.installed.base import NonZeroAgentExitCodeError
- from harbor.agents.installed.codex import Codex
-
- bridge_token = "bridge-client-token-secret"
- upstream_key = "upstream-nvidia-key-secret"
- agent = object.__new__(SkillEvaluatorNvidiaBuildCodex)
- agent.model_name = "nvidia/model"
- agent._extra_env = {"NVIDIA_API_KEY": upstream_key}
- agent._nvidia_build_bridge_origin = "http://127.0.0.1:43123"
- agent._nvidia_build_bridge_client_env = {"OPENAI_API_KEY": bridge_token}
-
- class Environment:
- async def exec_with_sensitive_env(self, **_kwargs: object) -> SimpleNamespace:
- return SimpleNamespace(
- return_code=17,
- stdout=f"provider echoed {bridge_token}",
- stderr=f"provider echoed {upstream_key}",
- )
-
- async def reject_logged_exec(
- _self: Codex,
- _environment: object,
- **_kwargs: object,
- ) -> SimpleNamespace:
- raise AssertionError("bridge credentials must bypass Harbor's logging wrapper")
-
- monkeypatch.setattr(Codex, "exec_as_agent", reject_logged_exec)
-
- with pytest.raises(NonZeroAgentExitCodeError) as exc_info:
- asyncio.run(
- agent.exec_as_agent(
- Environment(),
- command="codex exec -- test",
- env={"NVIDIA_API_KEY": "per-call-upstream-key-secret"},
- )
- )
-
- message = str(exc_info.value)
- assert message == "NVIDIA Build bridge client command failed with exit code 17"
- assert bridge_token not in message
- assert upstream_key not in message
+ assert scoped == [
+ {"OPENAI_API_KEY": "bridge-client-token-secret"},
+ {"OPENAI_API_KEY": "bridge-client-token-secret"},
+ ]
def test_nvidia_build_claude_bridge_configures_origin_and_full_model(
@@ -1218,7 +1544,7 @@ def test_nvidia_build_claude_bridge_configures_origin_and_full_model(
agent.model_name = "nvidia/meta/llama-3.1-8b-instruct"
calls: list[tuple[str, dict[str, str]]] = []
- class Environment:
+ class Environment(_NoopScopedExecEnvironment):
async def upload_file(self, _source: object, _destination: object) -> None:
return None
@@ -1496,85 +1822,719 @@ def wrap(argv: list[str], **_kwargs: object) -> list[str]:
@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
-def test_exec_forwards_strict_read_policy_to_sandbox(tmp_path: Path) -> None:
- environment = _local_environment(tmp_path)
- environment._strict_reads = True
- captured: dict[str, object] = {}
-
- class CaptureSandbox:
- plan = local_sandbox.SandboxPlan("none", "advisory-only", "capture")
+def test_real_local_exec_streams_stdout_and_stderr_through_harbor_callback(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[tuple[str, str]] = []
- @staticmethod
- def wrap(argv: list[str], **kwargs: object) -> list[str]:
- captured.update(kwargs)
- return argv
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
- environment._sandbox = CaptureSandbox()
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec("printf stdout-value; printf stderr-value >&2")
- result = asyncio.run(environment.exec("printf ok"))
+ result = asyncio.run(exercise())
- assert result.return_code == 0
- assert captured["strict_reads"] is True
+ assert result.stdout == "stdout-value"
+ assert result.stderr == "stderr-value"
+ assert "".join(text for text, stream in callback_chunks if stream == "stdout") == result.stdout
+ assert "".join(text for text, stream in callback_chunks if stream == "stderr") == result.stderr
@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
-@pytest.mark.parametrize("strict_reads", [False, True])
-def test_seatbelt_exec_uses_canonical_interpreter_for_sandbox_bootstrap(
- tmp_path: Path,
- monkeypatch: pytest.MonkeyPatch,
- strict_reads: bool,
-) -> None:
- venv_python = tmp_path / "venv" / "bin" / "python"
- venv_python.parent.mkdir(parents=True)
- venv_python.symlink_to(Path(sys.executable).resolve())
- monkeypatch.setattr(sys, "executable", str(venv_python))
- environment = _local_environment(tmp_path)
- environment._strict_reads = strict_reads
- captured: dict[str, object] = {}
-
- class CaptureSandbox:
- plan = local_sandbox.SandboxPlan("seatbelt", "kernel-macos", "capture")
-
- @staticmethod
- def wrap(argv: list[str], **_kwargs: object) -> list[str]:
- captured["argv"] = argv
- return argv
-
- environment._sandbox = CaptureSandbox()
+def test_local_streamed_nonzero_exit_preserves_output_and_return_code(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[tuple[str, str]] = []
+ secret = "nonzero-stream-secret"
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ 'printf "stdout=%s\\n" "$NONZERO_TOKEN"; printf "stderr=%s\\n" "$NONZERO_TOKEN" >&2; exit 7',
+ env={"NONZERO_TOKEN": secret},
+ )
- result = asyncio.run(environment.exec("printf ok"))
+ result = asyncio.run(exercise())
+ callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
+ callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")
- assert result.return_code == 0, result.stderr
- wrapped_argv = captured["argv"]
- assert isinstance(wrapped_argv, list)
- assert wrapped_argv[0] == str(Path(sys.executable).resolve())
+ assert result.return_code == 7
+ assert callback_stdout == result.stdout
+ assert callback_stderr == result.stderr
+ assert secret not in callback_stdout
+ assert secret not in callback_stderr
@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@pytest.mark.parametrize(
- ("backend", "strength", "strict_reads"),
+ "proxy_uri",
[
- ("bubblewrap", "kernel", True),
- ("none", "advisory-only", True),
+ "https://local-user:local-password@proxy.invalid:8443",
+ "https://local%2Duser:local%2Dpassword@proxy.invalid:8443",
+ "local-user:local-password@proxy.invalid:8443",
],
+ ids=["uri", "percent-encoded", "schemeless"],
)
-def test_other_sandbox_modes_preserve_venv_interpreter_for_bootstrap(
+def test_local_nonzero_exit_redacts_detached_proxy_components_from_callback_and_result(
tmp_path: Path,
- backend: str,
- strength: str,
- strict_reads: bool,
+ proxy_uri: str,
) -> None:
- environment = _local_environment(tmp_path)
- environment._strict_reads = strict_reads
- captured: dict[str, object] = {}
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[tuple[str, str]] = []
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ "printf 'proxy rejected local-user with local-password\\n'; exit 7",
+ env={"HTTPS_PROXY": proxy_uri},
+ )
- class CaptureSandbox:
- plan = local_sandbox.SandboxPlan(backend, strength, "capture")
+ result = asyncio.run(exercise())
+ rendered = "".join(text for text, _stream in callback_chunks) + (result.stdout or "") + (result.stderr or "")
- @staticmethod
- def wrap(argv: list[str], **_kwargs: object) -> list[str]:
- captured["argv"] = argv
- return argv
+ assert result.return_code == 7
+ assert "local-user" not in rendered
+ assert "local-password" not in rendered
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_real_local_exec_redacts_merged_secrets_across_byte_and_line_boundaries(
+ tmp_path: Path,
+) -> None:
+ persistent_secret = "persistent-first\npersistent-second"
+ scoped_secret = "scoped-secret-value"
+ per_call_secret = "per-call-secret-value"
+ environment = _initialized_local_environment(
+ tmp_path,
+ persistent_env={"PERSISTENT_TOKEN": persistent_secret},
+ )
+ callback_chunks: list[tuple[str, str]] = []
+ script = """
+import os
+import time
+
+values = (
+ (1, os.environ["PERSISTENT_TOKEN"]),
+ (2, os.environ["SCOPED_SECRET"]),
+ (1, os.environ["PER_CALL_KEY"]),
+ (2, "unicode-snowman-☃"),
+)
+for descriptor, value in values:
+ payload = value.encode("utf-8")
+ for byte in payload:
+ os.write(descriptor, bytes((byte,)))
+ time.sleep(0.001)
+ os.write(descriptor, b"\\n")
+"""
+ command = f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}"
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with (
+ environment.scoped_exec_env({"SCOPED_SECRET": scoped_secret}),
+ environment.scoped_output_callback(on_output),
+ ):
+ return await environment.exec(command, env={"PER_CALL_KEY": per_call_secret})
+
+ result = asyncio.run(exercise())
+ callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
+ callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")
+
+ assert callback_stdout == result.stdout
+ assert callback_stderr == result.stderr
+ assert "unicode-snowman-☃" in callback_stderr
+ for secret in (persistent_secret, scoped_secret, per_call_secret):
+ assert secret not in callback_stdout
+ assert secret not in callback_stderr
+ assert secret not in (result.stdout or "")
+ assert secret not in (result.stderr or "")
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_stream_redaction_handles_short_sensitive_and_marker_collision_values(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[str] = []
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ 'printf "%s|%s|%s" "$API_KEY" "$PUBLIC_LABEL" "$COLLISION_SECRET"',
+ env={
+ "API_KEY": "x",
+ "PUBLIC_LABEL": "ok",
+ "COLLISION_SECRET": "redacted",
+ },
+ )
+
+ result = asyncio.run(exercise())
+ callback_output = "".join(callback_chunks)
+
+ assert callback_output == result.stdout
+ assert "|ok|" in callback_output
+ assert "x" not in callback_output
+ assert "redacted" not in callback_output.lower()
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_stream_redacts_short_compact_credential_names_without_substring_false_positives(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[str] = []
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ 'printf "%s|%s|%s|%s" "$MYSECRET" "$AUTHENTICATION" "$MONKEY" "$KEYBOARD"',
+ env={
+ "MYSECRET": "pw",
+ "AUTHENTICATION": "id",
+ "MONKEY": "banana",
+ "KEYBOARD": "clacky",
+ },
+ )
+
+ result = asyncio.run(exercise())
+ callback_output = "".join(callback_chunks)
+
+ assert callback_output == result.stdout
+ assert "pw" not in callback_output
+ assert "id" not in callback_output
+ assert callback_output.endswith("banana|clacky")
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_stream_redacts_known_secret_patterns_across_reader_chunks(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[tuple[str, str]] = []
+ sk_secret = "".join(("sk-", "Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8")) # noqa: FLY002
+ nvapi_secret = "".join(("nvapi-", "Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8")) # noqa: FLY002
+ crsr_secret = "".join(("crsr_", "0123456789abcdef")) # noqa: FLY002
+ openshift_secret = "".join(("sha256~", "Abc.def_Ghi-jkl~mno")) # noqa: FLY002
+ jwt_secret = ".".join(("eyJ" + "A" * 20, "B" * 20, "C" * 20))
+ script = f"""
+import os
+import time
+
+values = (
+ (1, {sk_secret!r}),
+ (2, {nvapi_secret!r}),
+ (1, {crsr_secret!r}),
+ (2, {openshift_secret!r}),
+ (1, {jwt_secret!r}),
+ (2, "task-granularity"),
+)
+for descriptor, value in values:
+ for byte in value.encode("utf-8"):
+ os.write(descriptor, bytes((byte,)))
+ time.sleep(0.001)
+ os.write(descriptor, b"\\n")
+"""
+ command = f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}"
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(command)
+
+ result = asyncio.run(exercise())
+ callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
+ callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")
+
+ assert callback_stdout == result.stdout
+ assert callback_stderr == result.stderr
+ assert "task-granularity" in callback_stderr
+ for secret in (sk_secret, nvapi_secret, crsr_secret, openshift_secret, jwt_secret):
+ assert secret not in callback_stdout
+ assert secret not in callback_stderr
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_concurrent_callback_contexts_are_isolated(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ labels = tuple(f"callback-{index}" for index in range(10))
+ callback_outputs: dict[str, list[str]] = {label: [] for label in labels}
+
+ async def exercise() -> dict[str, object]:
+ await environment.start()
+
+ async def run(label: str) -> object:
+ async def on_output(text: str, _stream: str) -> None:
+ callback_outputs[label].append(text)
+
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(f"printf '{label}-first\\n'; sleep 0.05; printf '{label}-second\\n'")
+
+ results = await asyncio.gather(*(run(label) for label in labels))
+ return dict(zip(labels, results, strict=True))
+
+ results = asyncio.run(exercise())
+
+ for label in labels:
+ rendered = "".join(callback_outputs[label])
+ assert rendered == results[label].stdout
+ for other_label in labels:
+ if other_label != label:
+ assert other_label not in rendered
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_nested_callbacks_run_in_scope_order(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ calls: list[tuple[str, str, str]] = []
+
+ async def outer(text: str, stream: str) -> None:
+ calls.append(("outer", text, stream))
+
+ async def inner(text: str, stream: str) -> None:
+ calls.append(("inner", text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with (
+ environment.scoped_output_callback(outer),
+ environment.scoped_output_callback(inner),
+ ):
+ return await environment.exec("printf 'nested-output\\n'")
+
+ result = asyncio.run(exercise())
+
+ assert result.stdout == "nested-output\n"
+ assert calls == [
+ ("outer", "nested-output\n", "stdout"),
+ ("inner", "nested-output\n", "stdout"),
+ ]
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_callback_receives_complete_line_before_process_exit(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ first_line = asyncio.Event()
+
+ async def on_output(text: str, stream: str) -> None:
+ if text == "first-line\n" and stream == "stdout":
+ first_line.set()
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ task = asyncio.create_task(environment.exec("printf 'first-line\\n'; sleep 1; printf done"))
+ await asyncio.wait_for(first_line.wait(), timeout=0.5)
+ assert not task.done()
+ return await task
+
+ result = asyncio.run(exercise())
+
+ assert result.stdout == "first-line\ndone"
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_callback_streams_safe_partial_line_before_process_exit(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ partial_output = asyncio.Event()
+ callback_chunks: list[str] = []
+
+ async def on_output(text: str, stream: str) -> None:
+ if stream == "stdout":
+ callback_chunks.append(text)
+ if "safe-partial-output" in "".join(callback_chunks):
+ partial_output.set()
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ task = asyncio.create_task(environment.exec("printf safe-partial-output; sleep 1; printf done"))
+ await asyncio.wait_for(partial_output.wait(), timeout=0.5)
+ assert not task.done()
+ return await task
+
+ result = asyncio.run(exercise())
+
+ assert "".join(callback_chunks) == result.stdout == "safe-partial-outputdone"
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_streaming_preserves_exact_json_stdin_bootstrap(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ captured_payloads: list[bytes] = []
+ original_collect = environment._collect_streamed_output
+ per_call_env = {"PER_CALL_VALUE": "☃"}
+
+ async def capture_payload(
+ proc: asyncio.subprocess.Process,
+ stdin_data: bytes,
+ callback_output: object,
+ ) -> tuple[bytes, bytes]:
+ captured_payloads.append(stdin_data)
+ return await original_collect(
+ proc,
+ stdin_data,
+ callback_output, # type: ignore[arg-type]
+ )
+
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ async def exercise() -> tuple[object, bytes]:
+ await environment.start()
+ with (
+ environment.scoped_exec_env({"SCOPED_VALUE": "scoped"}),
+ environment.scoped_output_callback(on_output),
+ ):
+ expected_payload = json.dumps(environment._exec_env(per_call_env)).encode("utf-8")
+ environment._collect_streamed_output = capture_payload # type: ignore[method-assign]
+ result = await environment.exec('printf %s "$PER_CALL_VALUE"', env=per_call_env)
+ return result, expected_payload
+
+ result, expected_payload = asyncio.run(exercise())
+
+ assert result.stdout == "☃"
+ assert captured_payloads == [expected_payload]
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_exec_without_callback_uses_bounded_stream_collector(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ original_collect = environment._collect_streamed_output
+ callback_outputs: list[object] = []
+
+ async def capture_stream(
+ proc: asyncio.subprocess.Process,
+ stdin_data: bytes,
+ callback_output: object,
+ ) -> tuple[bytes, bytes]:
+ callback_outputs.append(callback_output)
+ return await original_collect(
+ proc,
+ stdin_data,
+ callback_output, # type: ignore[arg-type]
+ )
+
+ async def exercise() -> object:
+ await environment.start()
+ environment._collect_streamed_output = capture_stream # type: ignore[method-assign]
+ return await environment.exec("printf bounded-only")
+
+ result = asyncio.run(exercise())
+
+ assert callback_outputs == [None]
+ assert result.stdout == "bounded-only"
+ assert result.stderr == ""
+ assert result.return_code == 0
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_streaming_tolerates_child_closing_json_stdin_early(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+
+ class EarlyExitSandbox:
+ plan = local_sandbox.SandboxPlan("none", "advisory-only", "early-exit-test")
+
+ @staticmethod
+ def wrap(_argv: list[str], **_kwargs: object) -> list[str]:
+ return [sys.executable, "-c", "import os; os._exit(7)"]
+
+ environment._sandbox = EarlyExitSandbox()
+
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ async def exercise() -> object:
+ await environment.start()
+ environment._sandbox = EarlyExitSandbox()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec("ignored", env={"FILLER": "v" * (1024 * 1024)})
+
+ result = asyncio.run(exercise())
+
+ assert result.return_code == 7
+ assert result.stdout == ""
+ assert result.stderr == ""
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
+@pytest.mark.parametrize("error_type", [TimeoutError, _LocalCallbackBaseError, asyncio.CancelledError])
+def test_local_callback_exception_is_propagated_after_process_reap(
+ tmp_path: Path,
+ error_type: type[BaseException],
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_error = error_type("local callback failed")
+ secret = "local-callback-base-error-secret"
+ processes: list[asyncio.subprocess.Process] = []
+ create_subprocess_exec = asyncio.create_subprocess_exec
+ callback_chunks: list[str] = []
+ reaped_by_exec: list[bool] = []
+ caught_errors: list[BaseException] = []
+
+ async def capture_process(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await create_subprocess_exec(*args, **kwargs)
+ processes.append(process)
+ return process
+
+ async def failing_callback(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+ raise callback_error
+
+ async def exercise() -> None:
+ await environment.start()
+ try:
+ with (
+ pytest.MonkeyPatch.context() as patch,
+ environment.scoped_output_callback(failing_callback),
+ ):
+ patch.setattr(asyncio, "create_subprocess_exec", capture_process)
+ for _ in range(3):
+ with pytest.raises(error_type) as caught:
+ await environment.exec(
+ 'printf "%s\\n" "$CALLBACK_SECRET"; sleep 30',
+ env={"CALLBACK_SECRET": secret},
+ )
+ caught_errors.append(caught.value)
+ reaped_by_exec.append(processes[-1].returncode is not None)
+ finally:
+ for process in processes:
+ if process.returncode is None:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(process.pid, signal.SIGKILL)
+ await process.wait()
+
+ asyncio.run(exercise())
+
+ assert callback_chunks
+ assert secret not in "".join(callback_chunks)
+ assert secret not in str(callback_error)
+ assert caught_errors == [callback_error] * 3
+ assert len(processes) == 3
+ assert reaped_by_exec == [True] * 3
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
+def test_local_callback_error_remains_primary_when_cleanup_reports_failure(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_error = _LocalCallbackBaseError("primary callback failure")
+ secret = "synthetic cleanup report failure"
+ original_terminate = environment._terminate_process_tree
+ cleanup_calls = 0
+
+ async def failing_cleanup(
+ _proc: asyncio.subprocess.Process,
+ _communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
+ ) -> tuple[bytes, bytes]:
+ nonlocal cleanup_calls
+ cleanup_calls += 1
+ raise PermissionError(errno.EACCES, "Denied", secret)
+
+ async def failing_callback(_text: str, _stream: str) -> None:
+ raise callback_error
+
+ async def exercise() -> tuple[BaseException, bool, bool]:
+ await environment.start()
+ environment._terminate_process_tree = failing_cleanup # type: ignore[method-assign]
+ with (
+ environment.scoped_output_callback(failing_callback),
+ pytest.raises(_LocalCallbackBaseError) as caught,
+ ):
+ await environment.exec(
+ "printf 'callback-output\\n'; sleep 30",
+ env={"API_KEY": secret},
+ )
+ retained_before_retry = bool(environment._active_processes)
+ environment._terminate_process_tree = original_terminate # type: ignore[method-assign]
+ await environment.stop(delete=False)
+ return caught.value, retained_before_retry, not environment._active_processes
+
+ caught, retained_before_retry, released_after_retry = asyncio.run(exercise())
+
+ assert caught is callback_error
+ assert isinstance(caught.__cause__, RuntimeError)
+ assert caught.__cause__.__context__ is None
+ assert cleanup_calls == 1
+ assert retained_before_retry
+ assert released_after_retry
+ assert any("cleanup" in note.lower() for note in caught.__notes__)
+ assert secret not in "".join(caught.__notes__)
+ assert secret not in str(caught.__cause__)
+ assert secret not in "".join(traceback.format_exception(caught))
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
+def test_local_callback_error_message_receives_only_redacted_output(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ secret = "callback-error-message-secret"
+
+ async def failing_callback(text: str, _stream: str) -> None:
+ raise RuntimeError(f"consumer rejected: {text}")
+
+ async def exercise() -> RuntimeError:
+ await environment.start()
+ with (
+ environment.scoped_output_callback(failing_callback),
+ pytest.raises(RuntimeError, match="consumer rejected") as caught,
+ ):
+ await environment.exec(
+ 'printf "%s\\n" "$ERROR_TOKEN"; sleep 30',
+ env={"ERROR_TOKEN": secret},
+ )
+ return caught.value
+
+ error = asyncio.run(exercise())
+
+ assert secret not in str(error)
+ assert "consumer rejected" in str(error)
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
+def test_local_stream_collector_failure_is_propagated_after_process_reap(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ collector_error = RuntimeError("synthetic stream collector failure")
+ processes: list[asyncio.subprocess.Process] = []
+ create_subprocess_exec = asyncio.create_subprocess_exec
+ reaped_by_exec: list[bool] = []
+
+ async def capture_process(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await create_subprocess_exec(*args, **kwargs)
+ processes.append(process)
+ return process
+
+ async def failing_collector(*_args: object, **_kwargs: object) -> tuple[bytes, bytes]:
+ raise collector_error
+
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ async def exercise() -> BaseException:
+ await environment.start()
+ environment._collect_streamed_output = failing_collector # type: ignore[method-assign]
+ try:
+ with (
+ pytest.MonkeyPatch.context() as patch,
+ environment.scoped_output_callback(on_output),
+ pytest.raises(RuntimeError, match="synthetic stream collector failure") as caught,
+ ):
+ patch.setattr(asyncio, "create_subprocess_exec", capture_process)
+ await environment.exec("sleep 30")
+ reaped_by_exec.append(processes[0].returncode is not None)
+ return caught.value
+ finally:
+ for process in processes:
+ if process.returncode is None:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(process.pid, signal.SIGKILL)
+ await process.wait()
+
+ caught = asyncio.run(exercise())
+
+ assert caught is collector_error
+ assert caught.__cause__ is None
+ assert reaped_by_exec == [True]
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_exec_forwards_strict_read_policy_to_sandbox(tmp_path: Path) -> None:
+ environment = _local_environment(tmp_path)
+ environment._strict_reads = True
+ captured: dict[str, object] = {}
+
+ class CaptureSandbox:
+ plan = local_sandbox.SandboxPlan("none", "advisory-only", "capture")
+
+ @staticmethod
+ def wrap(argv: list[str], **kwargs: object) -> list[str]:
+ captured.update(kwargs)
+ return argv
+
+ environment._sandbox = CaptureSandbox()
+
+ result = asyncio.run(environment.exec("printf ok"))
+
+ assert result.return_code == 0
+ assert captured["strict_reads"] is True
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("strict_reads", [False, True])
+def test_seatbelt_exec_uses_canonical_interpreter_for_sandbox_bootstrap(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ strict_reads: bool,
+) -> None:
+ venv_python = tmp_path / "venv" / "bin" / "python"
+ venv_python.parent.mkdir(parents=True)
+ venv_python.symlink_to(Path(sys.executable).resolve())
+ monkeypatch.setattr(sys, "executable", str(venv_python))
+ environment = _local_environment(tmp_path)
+ environment._strict_reads = strict_reads
+ captured: dict[str, object] = {}
+
+ class CaptureSandbox:
+ plan = local_sandbox.SandboxPlan("seatbelt", "kernel-macos", "capture")
+
+ @staticmethod
+ def wrap(argv: list[str], **_kwargs: object) -> list[str]:
+ captured["argv"] = argv
+ return argv
+
+ environment._sandbox = CaptureSandbox()
+
+ result = asyncio.run(environment.exec("printf ok"))
+
+ assert result.return_code == 0, result.stderr
+ wrapped_argv = captured["argv"]
+ assert isinstance(wrapped_argv, list)
+ assert wrapped_argv[0] == str(Path(sys.executable).resolve())
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize(
+ ("backend", "strength", "strict_reads"),
+ [
+ ("bubblewrap", "kernel", True),
+ ("none", "advisory-only", True),
+ ],
+)
+def test_other_sandbox_modes_preserve_venv_interpreter_for_bootstrap(
+ tmp_path: Path,
+ backend: str,
+ strength: str,
+ strict_reads: bool,
+) -> None:
+ environment = _local_environment(tmp_path)
+ environment._strict_reads = strict_reads
+ captured: dict[str, object] = {}
+
+ class CaptureSandbox:
+ plan = local_sandbox.SandboxPlan(backend, strength, "capture")
+
+ @staticmethod
+ def wrap(argv: list[str], **_kwargs: object) -> list[str]:
+ captured["argv"] = argv
+ return argv
environment._sandbox = CaptureSandbox()
@@ -1650,81 +2610,738 @@ async def run_timeout() -> object:
result = asyncio.run(run_timeout())
- assert result.return_code == 124
- assert result.stdout == "stdout-before-timeout"
- assert "stderr-before-timeout" in (result.stderr or "")
- assert "Timed out" in (result.stderr or "")
- assert started.exists(), "the background descendant did not start before the timeout"
+ assert result.return_code == 124
+ assert result.stdout == "stdout-before-timeout"
+ assert "stderr-before-timeout" in (result.stderr or "")
+ assert "Timed out" in (result.stderr or "")
+ assert started.exists(), "the background descendant did not start before the timeout"
+
+ def process_exists(pid: int) -> bool:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ return True
+ return True
+
+ child_pid = int(child_pid_path.read_text())
+ deadline = time.monotonic() + 10
+ while process_exists(child_pid) and time.monotonic() < deadline:
+ time.sleep(0.05)
+ assert not process_exists(child_pid), "a background descendant survived the timeout kill"
+ assert not marker.exists(), "a background descendant wrote after the command timed out"
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_streamed_timeout_callback_matches_result_and_contains_descendants(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+ secret = "stream-timeout-secret-value"
+ child_pid_path = environment._workspace / "stream-timeout-child-pid"
+ callback_chunks: list[tuple[str, str]] = []
+ command = (
+ 'printf "%s\\n" "$TIMEOUT_TOKEN"; printf "stderr-before-timeout\\n" >&2; '
+ "(sleep 30) & printf '%s' \"$!\" > stream-timeout-child-pid; wait"
+ )
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ command,
+ env={"TIMEOUT_TOKEN": secret},
+ timeout_sec=1,
+ )
+
+ result = asyncio.run(exercise())
+ callback_stdout = "".join(text for text, stream in callback_chunks if stream == "stdout")
+ callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")
+
+ assert result.return_code == 124
+ assert callback_stdout == result.stdout
+ assert callback_stderr == result.stderr
+ assert callback_stderr == "stderr-before-timeout\nTimed out"
+ assert secret not in callback_stdout
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ with pytest.raises(ProcessLookupError):
+ os.kill(child_pid, 0)
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_local_timeout_redacts_detached_proxy_components_from_callback_and_result(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+ callback_chunks: list[tuple[str, str]] = []
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ "printf 'proxy rejected local-timeout-user with local-timeout-password\\n'; sleep 30",
+ env={
+ "HTTPS_PROXY": "https://local-timeout-user:local-timeout-password@proxy.invalid:8443",
+ },
+ timeout_sec=0.1,
+ )
+
+ result = asyncio.run(exercise())
+ rendered = "".join(text for text, _stream in callback_chunks) + (result.stdout or "") + (result.stderr or "")
+
+ assert result.return_code == 124
+ assert "local-timeout-user" not in rendered
+ assert "local-timeout-password" not in rendered
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("with_callback", [False, True])
+def test_timeout_diagnostic_cannot_synthesize_sensitive_value(
+ tmp_path: Path,
+ with_callback: bool,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ secret = "Timed out"
+ callback_chunks: list[tuple[str, str]] = []
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
+ with callback_scope:
+ return await environment.exec(
+ "sleep 30",
+ env={"TIMEOUT_SECRET": secret},
+ timeout_sec=0.1,
+ )
+
+ result = asyncio.run(exercise())
+ callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")
+
+ assert result.return_code == 124
+ assert secret not in (result.stderr or "")
+ assert secret not in callback_stderr
+ if with_callback:
+ assert callback_stderr == result.stderr
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("with_callback", [False, True])
+@pytest.mark.parametrize("secret", ["prefix\nTimed out", "\n"])
+def test_timeout_diagnostic_uses_the_live_stderr_redactor_across_its_boundary(
+ tmp_path: Path,
+ with_callback: bool,
+ secret: str,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[tuple[str, str]] = []
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
+ with callback_scope:
+ return await environment.exec(
+ "printf prefix >&2; sleep 30",
+ env={"TIMEOUT_SECRET": secret},
+ timeout_sec=0.1,
+ )
+
+ result = asyncio.run(exercise())
+ callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")
+
+ assert result.return_code == 124
+ assert secret not in (result.stderr or "")
+ assert secret not in callback_stderr
+ if with_callback:
+ assert callback_stderr == result.stderr
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_blocked_callback_does_not_replace_command_timeout_with_cancelled_error(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ release_callback = asyncio.Event()
+
+ async def blocked_callback(_text: str, _stream: str) -> None:
+ await release_callback.wait()
+
+ async def exercise() -> tuple[object, float]:
+ await environment.start()
+ started = time.monotonic()
+ with environment.scoped_output_callback(blocked_callback):
+ result = await environment.exec("printf callback-blocked; sleep 30", timeout_sec=0.1)
+ return result, time.monotonic() - started
+
+ result, elapsed = asyncio.run(exercise())
+
+ assert elapsed < 3
+ assert result.return_code == 124
+ assert "Timed out" in (result.stderr or "")
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_callback_suppressing_one_cleanup_cancellation_is_not_reentered_or_leaked(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import local_environment
+
+ environment = _initialized_local_environment(tmp_path)
+ monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
+ monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
+ monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.05)
+ callback_calls = 0
+
+ async def cancellation_suppressing_callback(_text: str, _stream: str) -> None:
+ nonlocal callback_calls
+ callback_calls += 1
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ # A callback can perform async cleanup after its first
+ # cancellation. A bounded second cancellation must finish that
+ # cleanup path; exec must never invoke it concurrently again.
+ await asyncio.Event().wait()
+
+ async def exercise() -> tuple[object, list[str]]:
+ await environment.start()
+ with environment.scoped_output_callback(cancellation_suppressing_callback):
+ result = await asyncio.wait_for(
+ environment.exec("printf callback-blocked; sleep 30", timeout_sec=0.02),
+ timeout=0.5,
+ )
+ await asyncio.sleep(0)
+ leaked = [
+ repr(task.get_coro())
+ for task in asyncio.all_tasks()
+ if task is not asyncio.current_task()
+ and not task.done()
+ and any(
+ name in repr(task.get_coro())
+ for name in ("_collect_streamed_output", "invoke_callback", "_cancel_task_repeatedly")
+ )
+ ]
+ return result, leaked
+
+ result, leaked = asyncio.run(exercise())
+
+ assert result.return_code == 124
+ assert callback_calls == 1
+ assert leaked == []
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("repeat_cancellation", [False, True])
+def test_cancellation_during_timeout_diagnostic_callback_reaps_callback_tasks(
+ tmp_path: Path,
+ repeat_cancellation: bool,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import local_environment
+
+ environment = _initialized_local_environment(tmp_path)
+ monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
+ monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
+ monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.05)
+ callback_started = asyncio.Event()
+ callback_calls = 0
+
+ async def blocked_callback(_text: str, _stream: str) -> None:
+ nonlocal callback_calls
+ callback_calls += 1
+ callback_started.set()
+ await asyncio.Event().wait()
+
+ async def exercise() -> tuple[BaseException | None, list[str]]:
+ await environment.start()
+ with environment.scoped_output_callback(blocked_callback):
+ task = asyncio.create_task(environment.exec("sleep 30", timeout_sec=0.02))
+ await asyncio.wait_for(callback_started.wait(), timeout=2)
+ task.cancel()
+ if repeat_cancellation:
+ await asyncio.sleep(0)
+ task.cancel()
+ outcome: BaseException | None = None
+ try:
+ await asyncio.wait_for(task, timeout=1)
+ except BaseException as exc:
+ outcome = exc
+ await asyncio.sleep(0)
+ leaked = [
+ repr(pending.get_coro())
+ for pending in asyncio.all_tasks()
+ if pending is not asyncio.current_task()
+ and not pending.done()
+ and any(
+ name in repr(pending.get_coro()) for name in ("finish", "invoke_callback", "_cancel_task_repeatedly")
+ )
+ ]
+ return outcome, leaked
+
+ outcome, leaked = asyncio.run(exercise())
+
+ assert isinstance(outcome, asyncio.CancelledError)
+ assert callback_calls == 1
+ assert leaked == []
+ assert not environment._active_processes
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("repeat_cancellation", [False, True])
+def test_cancellation_during_timeout_callback_cleanup_is_not_swallowed(
+ tmp_path: Path,
+ repeat_cancellation: bool,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import local_environment
+
+ environment = _initialized_local_environment(tmp_path)
+ monkeypatch.setattr(local_environment, "_REAP_TERM_SECONDS", 0.01)
+ monkeypatch.setattr(local_environment, "_REAP_KILL_SECONDS", 0.01)
+ monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.05)
+ callback_cleanup_started = asyncio.Event()
+
+ async def cleanup_awaiting_callback(_text: str, _stream: str) -> None:
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ callback_cleanup_started.set()
+ await asyncio.Event().wait()
+
+ async def exercise() -> tuple[BaseException | object, list[str]]:
+ await environment.start()
+ with environment.scoped_output_callback(cleanup_awaiting_callback):
+ task = asyncio.create_task(environment.exec("sleep 30", timeout_sec=0.02))
+ # This is set by the timeout finalizer's own first cancellation,
+ # proving caller cancellation lands in its no-exception cleanup
+ # path rather than the preceding bounded wait.
+ await asyncio.wait_for(callback_cleanup_started.wait(), timeout=2)
+ task.cancel()
+ if repeat_cancellation:
+ await asyncio.sleep(0)
+ task.cancel()
+ try:
+ outcome: BaseException | object = await asyncio.wait_for(task, timeout=1)
+ except BaseException as exc:
+ outcome = exc
+ await asyncio.sleep(0)
+ leaked = [
+ repr(pending.get_coro())
+ for pending in asyncio.all_tasks()
+ if pending is not asyncio.current_task()
+ and not pending.done()
+ and any(
+ name in repr(pending.get_coro()) for name in ("finish", "invoke_callback", "_cancel_task_repeatedly")
+ )
+ ]
+ return outcome, leaked
+
+ outcome, leaked = asyncio.run(exercise())
+
+ assert isinstance(outcome, asyncio.CancelledError)
+ assert leaked == []
+ assert not environment._active_processes
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_large_dense_short_secret_has_the_same_completed_outcome_with_callback(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ script = "import os; os.write(1, b'x' * 2_000_000)"
+ command = f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}"
+
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ async def exercise(with_callback: bool) -> object:
+ await environment.start()
+ callback_scope = environment.scoped_output_callback(on_output) if with_callback else contextlib.nullcontext()
+ with callback_scope:
+ return await environment.exec(command, env={"API_KEY": "x"}, timeout_sec=1.5)
+
+ without_callback = asyncio.run(exercise(False))
+ with_callback = asyncio.run(exercise(True))
+
+ assert without_callback.return_code == with_callback.return_code == 0
+ assert without_callback.stdout == with_callback.stdout
+ assert "x" not in (with_callback.stdout or "")
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_streamed_exec_stays_bounded_when_output_closes_before_process_exit(tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[tuple[str, str]] = []
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> tuple[object, float]:
+ await environment.start()
+ started = time.monotonic()
+ with environment.scoped_output_callback(on_output):
+ result = await environment.exec("exec 1>&- 2>&-; sleep 30", timeout_sec=1)
+ return result, time.monotonic() - started
+
+ result, elapsed = asyncio.run(exercise())
+
+ assert elapsed < 3
+ assert result.return_code == 124
+ assert result.stdout == ""
+ assert result.stderr == "Timed out"
+ assert callback_chunks == [("Timed out", "stderr")]
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_exec_cancellation_terminates_background_descendants(tmp_path: Path) -> None:
+ environment = _local_environment(tmp_path)
+ child_ready = environment._workspace / "cancel-child-ready"
+ child_pid_path = environment._workspace / "cancel-child-pid"
+ marker = environment._workspace / "cancel-child-survived"
+ command = (
+ "(printf ready > cancel-child-ready; sleep 30; "
+ "printf survived > cancel-child-survived) & "
+ "printf '%s' \"$!\" > cancel-child-pid; wait"
+ )
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+
+ def process_exists(pid: int) -> bool:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ return True
+ return True
+
+ async def run_cancelled() -> None:
+ task = asyncio.create_task(environment.exec(command))
+ child_pid: int | None = None
+ for _ in range(500):
+ if child_ready.exists() and child_pid_path.exists():
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ break
+ if task.done():
+ pytest.fail(f"command exited before cancellation: {task.result()}")
+ await asyncio.sleep(0.01)
+ assert child_pid is not None, "the background descendant did not start before cancellation"
+
+ try:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=5)
+ for _ in range(200):
+ if not process_exists(child_pid):
+ break
+ await asyncio.sleep(0.01)
+ assert not process_exists(child_pid), "the background descendant survived command cancellation"
+ finally:
+ if process_exists(child_pid):
+ with contextlib.suppress(ProcessLookupError):
+ os.kill(child_pid, signal.SIGKILL)
+
+ asyncio.run(run_cancelled())
+
+ assert not marker.exists(), "a background descendant wrote after command cancellation"
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("repeat_cancellation", [False, True])
+def test_streamed_exec_cancellation_reaps_descendants(
+ tmp_path: Path,
+ repeat_cancellation: bool,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+ secret = "stream-cancel-secret-value"
+ child_pid_path = environment._workspace / "stream-cancel-child-pid"
+ callback_started = asyncio.Event()
+ callback_chunks: list[str] = []
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+ callback_started.set()
+
+ async def exercise() -> int:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ task = asyncio.create_task(
+ environment.exec(
+ 'printf "%s\\n" "$CANCEL_TOKEN"; (sleep 30) & printf \'%s\' "$!" > stream-cancel-child-pid; wait',
+ env={"CANCEL_TOKEN": secret},
+ )
+ )
+ await asyncio.wait_for(callback_started.wait(), timeout=5)
+ for _ in range(500):
+ if child_pid_path.exists():
+ break
+ await asyncio.sleep(0.01)
+ assert child_pid_path.exists()
+ task.cancel()
+ if repeat_cancellation:
+ await asyncio.sleep(0)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=5)
+ return int(child_pid_path.read_text(encoding="ascii"))
+
+ child_pid = asyncio.run(exercise())
+
+ assert callback_chunks
+ assert secret not in "".join(callback_chunks)
+ with pytest.raises(ProcessLookupError):
+ os.kill(child_pid, 0)
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("repeat_cancellation", [False, True])
+def test_streamed_exec_cancellation_during_final_flush_reaps_descendants(
+ tmp_path: Path,
+ repeat_cancellation: bool,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+ child_pid_path = environment._workspace / "flush-cancel-child-pid"
+ callback_started = asyncio.Event()
+ command = (
+ "(trap '' TERM; exec >/dev/null 2>&1; sleep 30) & "
+ f"printf '%s' \"$!\" > {shlex.quote(child_pid_path.name)}; "
+ # A lone known-token prefix stays buffered until final redactor flush.
+ "printf s"
+ )
+
+ async def blocked_callback(_text: str, _stream: str) -> None:
+ callback_started.set()
+ await asyncio.Event().wait()
+
+ async def exercise() -> int:
+ await environment.start()
+ with environment.scoped_output_callback(blocked_callback):
+ task = asyncio.create_task(environment.exec(command))
+ await asyncio.wait_for(callback_started.wait(), timeout=5)
+ assert child_pid_path.exists()
+ assert not environment._active_processes
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ with pytest.raises(ProcessLookupError):
+ os.kill(child_pid, 0)
+ task.cancel()
+ if repeat_cancellation:
+ await asyncio.sleep(0)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=5)
+ return int(child_pid_path.read_text(encoding="ascii"))
+
+ child_pid: int | None = None
+ try:
+ child_pid = asyncio.run(exercise())
+ with pytest.raises(ProcessLookupError):
+ os.kill(child_pid, 0)
+ finally:
+ if child_pid is None and child_pid_path.exists():
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ if child_pid is not None:
+ with contextlib.suppress(ProcessLookupError):
+ os.kill(child_pid, signal.SIGKILL)
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("repeat_cancellation", [False, True])
+def test_cancellation_during_timeout_reap_preserves_cancellation_and_containment(
+ tmp_path: Path,
+ repeat_cancellation: bool,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+ child_pid_path = environment._workspace / "timeout-reap-cancel-child-pid"
+ reap_entered = asyncio.Event()
+ original_terminate = environment._terminate_process_tree
+ command = (
+ "(trap '' TERM; exec >/dev/null 2>&1; sleep 30) & "
+ f"printf '%s' \"$!\" > {shlex.quote(child_pid_path.name)}; wait"
+ )
+
+ async def observed_terminate(
+ proc: asyncio.subprocess.Process,
+ communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
+ ) -> tuple[bytes, bytes]:
+ reap_entered.set()
+ return await original_terminate(proc, communication)
+
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ async def exercise() -> tuple[BaseException | None, int, bool]:
+ await environment.start()
+ environment._terminate_process_tree = observed_terminate # type: ignore[method-assign]
+ with environment.scoped_output_callback(on_output):
+ task = asyncio.create_task(environment.exec(command, timeout_sec=0.5))
+ await asyncio.wait_for(reap_entered.wait(), timeout=5)
+ assert child_pid_path.exists()
+ task.cancel()
+ if repeat_cancellation:
+ await asyncio.sleep(0)
+ task.cancel()
+ outcome: BaseException | None = None
+ try:
+ await asyncio.wait_for(task, timeout=5)
+ except BaseException as exc:
+ outcome = exc
+ retained_after_exec = bool(environment._active_processes)
+ if retained_after_exec:
+ await environment.stop(delete=False)
+ return outcome, int(child_pid_path.read_text(encoding="ascii")), retained_after_exec
+
+ child_pid: int | None = None
+ try:
+ outcome, child_pid, retained = asyncio.run(exercise())
+ assert isinstance(outcome, asyncio.CancelledError)
+ assert not retained
+ with pytest.raises(ProcessLookupError):
+ os.kill(child_pid, 0)
+ finally:
+ if child_pid is None and child_pid_path.exists():
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ if child_pid is not None:
+ with contextlib.suppress(ProcessLookupError):
+ os.kill(child_pid, signal.SIGKILL)
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("repeat_cancellation", [False, True])
+def test_cancellation_remains_primary_when_timeout_reap_fails(
+ tmp_path: Path,
+ repeat_cancellation: bool,
+) -> None:
+ from skillevaluator.tier3.harbor import local_environment
+
+ environment = _initialized_local_environment(tmp_path)
+ original_terminate = environment._terminate_process_tree
+ reap_entered = asyncio.Event()
+ release_failure = asyncio.Event()
+ secret = "timeout-reap-cleanup-secret"
+
+ async def failing_terminate(
+ _proc: asyncio.subprocess.Process,
+ _communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
+ ) -> tuple[bytes, bytes]:
+ async def fail_after_release() -> tuple[bytes, bytes]:
+ await release_failure.wait()
+ raise PermissionError(errno.EACCES, "Denied", secret)
+
+ cleanup = asyncio.create_task(fail_after_release())
+ reap_entered.set()
+ return await local_environment._await_task_uninterruptibly(cleanup)
+
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ async def exercise() -> tuple[BaseException | None, bool]:
+ await environment.start()
+ environment._terminate_process_tree = failing_terminate # type: ignore[method-assign]
+ with environment.scoped_output_callback(on_output):
+ task = asyncio.create_task(
+ environment.exec(
+ "sleep 30",
+ env={"API_KEY": secret},
+ timeout_sec=0.02,
+ )
+ )
+ await asyncio.wait_for(reap_entered.wait(), timeout=2)
+ task.cancel()
+ if repeat_cancellation:
+ await asyncio.sleep(0)
+ task.cancel()
+ release_failure.set()
+ outcome: BaseException | None = None
+ try:
+ await asyncio.wait_for(task, timeout=1)
+ except BaseException as exc:
+ outcome = exc
+ retained = bool(environment._active_processes)
+ environment._terminate_process_tree = original_terminate # type: ignore[method-assign]
+ await environment.stop(delete=False)
+ return outcome, retained
- def process_exists(pid: int) -> bool:
- try:
- os.kill(pid, 0)
- except ProcessLookupError:
- return False
- except PermissionError:
- return True
- return True
+ outcome, retained = asyncio.run(exercise())
- child_pid = int(child_pid_path.read_text())
- deadline = time.monotonic() + 10
- while process_exists(child_pid) and time.monotonic() < deadline:
- time.sleep(0.05)
- assert not process_exists(child_pid), "a background descendant survived the timeout kill"
- assert not marker.exists(), "a background descendant wrote after the command timed out"
+ assert isinstance(outcome, asyncio.CancelledError)
+ assert isinstance(outcome.__cause__, RuntimeError)
+ assert outcome.__cause__.__context__ is None
+ assert retained
+ assert secret not in str(outcome.__cause__)
+ assert secret not in "".join(traceback.format_exception(outcome))
+ assert not environment._active_processes
@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
-def test_exec_cancellation_terminates_background_descendants(tmp_path: Path) -> None:
- environment = _local_environment(tmp_path)
- child_ready = environment._workspace / "cancel-child-ready"
- child_pid_path = environment._workspace / "cancel-child-pid"
- marker = environment._workspace / "cancel-child-survived"
+@pytest.mark.parametrize("lifecycle", ["timeout", "cancel", "repeat-cancel"])
+def test_streamed_exec_escalates_after_launcher_exits_with_term_ignoring_descendant(
+ tmp_path: Path,
+ lifecycle: str,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
+ child_pid_path = environment._workspace / f"term-ignoring-{lifecycle}.pid"
command = (
- "(printf ready > cancel-child-ready; sleep 30; "
- "printf survived > cancel-child-survived) & "
- "printf '%s' \"$!\" > cancel-child-pid; wait"
+ "(trap '' TERM; exec >/dev/null 2>&1; sleep 30) & "
+ f"printf '%s' \"$!\" > {shlex.quote(child_pid_path.name)}; "
+ "printf ready; wait"
)
- environment._local_command_guardrail_reason = lambda *_args: "" # type: ignore[method-assign]
-
- def process_exists(pid: int) -> bool:
- try:
- os.kill(pid, 0)
- except ProcessLookupError:
- return False
- except PermissionError:
- return True
- return True
-
- async def run_cancelled() -> None:
- task = asyncio.create_task(environment.exec(command))
- child_pid: int | None = None
- for _ in range(500):
- if child_ready.exists() and child_pid_path.exists():
- child_pid = int(child_pid_path.read_text(encoding="ascii"))
- break
- if task.done():
- pytest.fail(f"command exited before cancellation: {task.result()}")
- await asyncio.sleep(0.01)
- assert child_pid is not None, "the background descendant did not start before cancellation"
- try:
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ async def exercise() -> object | None:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ # Leave enough startup margin for loaded CI hosts to launch the
+ # stdin bootstrap and create the descendant before timeout cleanup.
+ task = asyncio.create_task(environment.exec(command, timeout_sec=1.0 if lifecycle == "timeout" else None))
+ for _ in range(500):
+ if child_pid_path.exists():
+ break
+ await asyncio.sleep(0.01)
+ assert child_pid_path.exists()
+ if lifecycle == "timeout":
+ result = await asyncio.wait_for(task, timeout=5)
+ assert result.return_code == 124
+ return result
task.cancel()
+ if lifecycle == "repeat-cancel":
+ await asyncio.sleep(0)
+ task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=5)
- for _ in range(200):
- if not process_exists(child_pid):
- break
- await asyncio.sleep(0.01)
- assert not process_exists(child_pid), "the background descendant survived command cancellation"
- finally:
- if process_exists(child_pid):
+ return None
+
+ child_pid: int | None = None
+ try:
+ asyncio.run(exercise())
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ with pytest.raises(ProcessLookupError):
+ os.kill(child_pid, 0)
+ finally:
+ if child_pid is None and child_pid_path.exists():
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ if child_pid is not None:
+ try:
+ os.kill(child_pid, 0)
+ except ProcessLookupError:
+ pass
+ else:
with contextlib.suppress(ProcessLookupError):
os.kill(child_pid, signal.SIGKILL)
- asyncio.run(run_cancelled())
-
- assert not marker.exists(), "a background descendant wrote after command cancellation"
-
@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_process_tree_cleanup_is_bounded_and_escalates(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -1750,6 +3367,75 @@ async def run_cleanup() -> tuple[bytes, bytes]:
assert signals == [signal.SIGTERM, signal.SIGKILL]
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_process_tree_cleanup_suppresses_permission_race_after_leader_exit(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class FakeProcess:
+ pid = 4242
+ returncode = None
+
+ async def communication() -> tuple[bytes, bytes]:
+ return b"stdout", b"stderr"
+
+ monkeypatch.setattr(
+ os,
+ "killpg",
+ lambda *_args: (_ for _ in ()).throw(PermissionError("leader exited")),
+ )
+ monkeypatch.setattr(
+ os,
+ "getpgid",
+ lambda _pid: (_ for _ in ()).throw(ProcessLookupError()),
+ )
+
+ async def exercise() -> tuple[bytes, bytes]:
+ task = asyncio.create_task(communication())
+ return await SkillEvaluatorLocalEnvironment._terminate_process_tree( # type: ignore[arg-type]
+ FakeProcess(),
+ task,
+ )
+
+ result = asyncio.run(exercise())
+
+ assert result == (b"stdout", b"stderr")
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_process_tree_cleanup_propagates_live_permission_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class FakeProcess:
+ pid = 4242
+ returncode = None
+
+ async def communication() -> tuple[bytes, bytes]:
+ await asyncio.Event().wait()
+ return b"", b""
+
+ monkeypatch.setattr(
+ os,
+ "killpg",
+ lambda *_args: (_ for _ in ()).throw(PermissionError("live process denied")),
+ )
+ monkeypatch.setattr(os, "getpgid", lambda pid: pid)
+
+ async def exercise() -> None:
+ task = asyncio.create_task(communication())
+ try:
+ with pytest.raises(PermissionError, match="live process denied"):
+ await SkillEvaluatorLocalEnvironment._terminate_process_tree( # type: ignore[arg-type]
+ FakeProcess(),
+ task,
+ )
+ finally:
+ task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
+
+ asyncio.run(exercise())
+
+
@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_process_tree_cleanup_stays_bounded_when_communication_ignores_cancellation(
monkeypatch: pytest.MonkeyPatch,
@@ -1761,16 +3447,15 @@ def test_process_tree_cleanup_stays_bounded_when_communication_ignores_cancellat
monkeypatch.setattr(local_environment, "_REAP_CANCEL_SECONDS", 0.01, raising=False)
monkeypatch.setattr(local_environment.os, "killpg", lambda *_args: None)
- async def run_cleanup() -> bool:
+ async def run_cleanup() -> tuple[bool, bool]:
started = asyncio.Event()
- release = asyncio.Event()
async def stubborn_communication() -> tuple[bytes, bytes]:
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
- await release.wait()
+ await asyncio.Event().wait()
return b"", b""
communication = asyncio.create_task(stubborn_communication())
@@ -1788,12 +3473,10 @@ class FakeProcess:
)
done, _pending = await asyncio.wait({cleanup}, timeout=0.1)
finished_within_bound = cleanup in done
- release.set()
- await communication
await cleanup
- return finished_within_bound
+ return finished_within_bound, communication.cancelled()
- assert asyncio.run(run_cleanup()) is True
+ assert asyncio.run(run_cleanup()) == (True, True)
def test_stop_reaps_all_tracked_processes(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
@@ -1857,6 +3540,244 @@ async def delayed_create(*args: object, **kwargs: object) -> asyncio.subprocess.
asyncio.run(run_cancelled_during_create())
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_exec_cancellation_does_not_wait_forever_for_uncooperative_process_creation(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import local_environment
+
+ environment = _local_environment(tmp_path)
+ create_subprocess_exec = asyncio.create_subprocess_exec
+ monkeypatch.setattr(local_environment, "_CREATION_CANCEL_SECONDS", 0.02, raising=False)
+
+ async def exercise() -> tuple[bool, asyncio.subprocess.Process]:
+ process_created = asyncio.Event()
+ release_creation = asyncio.Event()
+ created: list[asyncio.subprocess.Process] = []
+
+ async def uncooperative_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await create_subprocess_exec(*args, **kwargs)
+ created.append(process)
+ process_created.set()
+ try:
+ await release_creation.wait()
+ except asyncio.CancelledError:
+ await release_creation.wait()
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", uncooperative_create)
+ task = asyncio.create_task(environment.exec("sleep 30"))
+ await asyncio.wait_for(process_created.wait(), timeout=5)
+ task.cancel()
+ await asyncio.sleep(0)
+ task.cancel()
+ done, _pending = await asyncio.wait({task}, timeout=0.2)
+ returned_within_bound = task in done
+ release_creation.set()
+ if task not in done:
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ else:
+ with pytest.raises(asyncio.CancelledError):
+ task.result()
+ for _ in range(500):
+ if created[0].returncode is not None:
+ break
+ await asyncio.sleep(0.01)
+ return returned_within_bound, created[0]
+
+ returned_within_bound, process = asyncio.run(exercise())
+
+ assert returned_within_bound
+ assert process.returncode is not None
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_stop_fails_closed_until_withheld_process_creation_is_reaped(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import local_environment
+
+ environment = _local_environment(tmp_path)
+ create_subprocess_exec = asyncio.create_subprocess_exec
+ monkeypatch.setattr(local_environment, "_CREATION_CANCEL_SECONDS", 0.02)
+ created: list[asyncio.subprocess.Process] = []
+
+ async def exercise() -> tuple[asyncio.subprocess.Process, asyncio.subprocess.Process, bool, bool]:
+ process_created = asyncio.Event()
+ release_creation = asyncio.Event()
+
+ active_process = await create_subprocess_exec(
+ "bash",
+ "-c",
+ "sleep 30",
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ start_new_session=True,
+ )
+ active_communication = asyncio.create_task(active_process.communicate())
+ environment._active_processes[active_process] = active_communication
+
+ async def withheld_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await create_subprocess_exec(*args, **kwargs)
+ created.append(process)
+ process_created.set()
+ await release_creation.wait()
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", withheld_create)
+ exec_task = asyncio.create_task(environment.exec("sleep 30"))
+ await asyncio.wait_for(process_created.wait(), timeout=5)
+ exec_task.cancel()
+ await asyncio.sleep(0)
+ exec_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(exec_task, timeout=0.2)
+
+ with pytest.raises(RuntimeError, match="could not confirm process creation containment"):
+ await asyncio.wait_for(environment.stop(delete=True), timeout=0.2)
+ root_preserved_while_unresolved = environment._root.exists()
+ process_alive_while_stop_failed = created[0].returncode is None
+ assert active_process.returncode is not None
+ assert active_process not in environment._active_processes
+
+ release_creation.set()
+ for _ in range(500):
+ if created[0].returncode is not None and not environment._creation_cleanups:
+ break
+ await asyncio.sleep(0.01)
+ await environment.stop(delete=True)
+ return created[0], active_process, root_preserved_while_unresolved, process_alive_while_stop_failed
+
+ process: asyncio.subprocess.Process | None = None
+ active_process: asyncio.subprocess.Process | None = None
+ try:
+ process, active_process, root_preserved, process_was_alive = asyncio.run(exercise())
+ assert root_preserved
+ assert process_was_alive
+ assert process.returncode is not None
+ assert not environment._root.exists()
+ assert not environment._pending_creations
+ assert not environment._creation_cleanups
+ finally:
+ leaked_process = process or (created[0] if created else None)
+ for process_to_reap in (leaked_process, active_process):
+ if process_to_reap is not None and process_to_reap.returncode is None:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(process_to_reap.pid, signal.SIGKILL)
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize("secret", ["PermissionError", "Local process creation cleanup failed"])
+def test_failed_late_creation_cleanup_retains_process_for_stop_retry(
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+ secret: str,
+) -> None:
+ environment = _local_environment(tmp_path)
+ original_terminate = environment._terminate_process_tree
+ process: asyncio.subprocess.Process | None = None
+ caplog.set_level(logging.ERROR, logger="asyncio")
+
+ async def exercise() -> tuple[bool, bool, bool]:
+ nonlocal process
+ process = await asyncio.create_subprocess_exec(
+ "bash",
+ "-c",
+ "sleep 30",
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ start_new_session=True,
+ )
+ creation = asyncio.create_task(asyncio.sleep(0, result=process))
+ environment._pending_creations.add(creation)
+ cleanup_attempts = 0
+
+ async def fail_twice_then_reap(
+ proc: asyncio.subprocess.Process,
+ communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
+ ) -> tuple[bytes, bytes]:
+ nonlocal cleanup_attempts
+ cleanup_attempts += 1
+ if cleanup_attempts <= 2:
+ raise PermissionError(errno.EACCES, "Denied", secret)
+ return await original_terminate(proc, communication)
+
+ environment._terminate_process_tree = fail_twice_then_reap # type: ignore[method-assign]
+ cleanup = environment._schedule_creation_cleanup(creation, secret_values={secret})
+ with pytest.raises(PermissionError):
+ await cleanup
+ await asyncio.sleep(0)
+ retained_after_late_failure = process in environment._active_processes
+
+ with pytest.raises(RuntimeError, match="cleanup is still pending"):
+ await environment.start()
+ with pytest.raises(RuntimeError, match="could not confirm process creation containment") as caught:
+ await environment.stop(delete=True)
+ first_stop_was_redacted = secret not in str(caught.value) and environment._root.exists()
+
+ await environment.stop(delete=True)
+ return retained_after_late_failure, first_stop_was_redacted, process.returncode is not None
+
+ try:
+ retained, first_stop_redacted, reaped = asyncio.run(exercise())
+ assert retained
+ assert first_stop_redacted
+ assert secret not in caplog.text
+ assert reaped
+ assert not environment._active_processes
+ assert not environment._creation_cleanup_errors
+ assert not environment._root.exists()
+ finally:
+ if process is not None and process.returncode is None:
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(process.pid, signal.SIGKILL)
+
+
+@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_creation_completing_during_stop_redacts_cleanup_failure(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _local_environment(tmp_path)
+ create_subprocess_exec = asyncio.create_subprocess_exec
+ original_terminate = environment._terminate_process_tree
+ secret = "stop-race-cleanup-secret"
+
+ async def stop_after_create(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await create_subprocess_exec(*args, **kwargs)
+ environment._stop_requested = True
+ return process
+
+ async def fail_before_reap(
+ _proc: asyncio.subprocess.Process,
+ _communication: asyncio.Task[tuple[bytes, bytes]] | None = None,
+ ) -> tuple[bytes, bytes]:
+ raise PermissionError(errno.EACCES, "Denied", secret)
+
+ async def exercise() -> tuple[BaseException, bool]:
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", stop_after_create)
+ environment._terminate_process_tree = fail_before_reap # type: ignore[method-assign]
+ with pytest.raises(RuntimeError) as caught:
+ await environment.exec("sleep 30", env={"API_KEY": secret})
+ retained = bool(environment._active_processes)
+ environment._terminate_process_tree = original_terminate # type: ignore[method-assign]
+ await environment.stop(delete=False)
+ return caught.value, retained
+
+ caught, retained = asyncio.run(exercise())
+
+ assert retained
+ assert caught.__context__ is None
+ assert secret not in str(caught)
+ assert secret not in "".join(traceback.format_exception(caught))
+ assert not environment._active_processes
+
+
@pytest.mark.skipif(os.name != "posix", reason=_NATIVE_WINDOWS_LOCAL_REASON)
def test_repeated_cancellation_during_process_creation_still_reaps_launcher(
tmp_path: Path,
@@ -1943,35 +3864,131 @@ class FakeProcess:
@pytest.mark.parametrize(
- "name",
+ "name",
+ [
+ "LD_PRELOAD",
+ "LD_AUDIT",
+ "LD_LIBRARY_PATH",
+ "DYLD_INSERT_LIBRARIES",
+ "PYTHONPATH",
+ "PYTHONHOME",
+ "BASH_ENV",
+ "ENV",
+ "ZDOTDIR",
+ "RUBYOPT",
+ "PERL5OPT",
+ "NODE_OPTIONS",
+ ],
+)
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_runtime_injection_env_is_blocked_before_launcher(name: str, tmp_path: Path) -> None:
+ environment = _local_environment(tmp_path)
+
+ async def unexpected_launch(*_args: object, **_kwargs: object) -> None:
+ pytest.fail("loader-controlled task environment reached the host launcher")
+
+ with pytest.MonkeyPatch.context() as patch:
+ patch.setattr(asyncio, "create_subprocess_exec", unexpected_launch)
+ result = asyncio.run(environment.exec("true", env={name: "attacker-controlled"}))
+
+ assert result.return_code == 126
+ assert name in (result.stderr or "")
+
+
+@pytest.mark.parametrize("name", ["BASH_ENV", "ENV", "NODE_OPTIONS", "PYTHONPATH"])
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+def test_empty_evaluator_loader_reset_is_removed_before_launcher(name: str, tmp_path: Path) -> None:
+ environment = _initialized_local_environment(tmp_path)
+
+ async def exercise() -> object:
+ await environment.start()
+ try:
+ return await environment.exec(f'test -z "${{{name}+x}}"', env={name: ""})
+ finally:
+ await environment.stop(delete=True)
+
+ result = asyncio.run(exercise())
+
+ assert result.return_code == 0, result.stderr
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize(
+ ("command", "cwd", "env", "secret"),
+ [
+ ("true", "/tmp/ordinary-secret-path", {"API_KEY": "/tmp/ordinary-secret-path"}, "/tmp/ordinary-secret-path"),
+ ("true", "/tmp/sk-Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8", None, "sk-Ab1Cd2Ef3Gh4Ij5Kl6Mn7Op8"),
+ ('touch "$SECRET_PATH"', None, {"SECRET_PATH": "/tmp/guardrail-secret-path"}, "/tmp/guardrail-secret-path"),
+ ("rm -rf /", None, {"API_KEY": "Local mode command blocked"}, "Local mode command blocked"),
+ ('touch "$API_KEY"', None, {"API_KEY": "/x"}, "/x"),
+ ],
+)
+def test_prelaunch_diagnostics_are_streamed_and_redacted_once(
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+ command: str,
+ cwd: str | None,
+ env: dict[str, str] | None,
+ secret: str,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[tuple[str, str]] = []
+ caplog.set_level(logging.WARNING)
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(command, cwd=cwd, env=env)
+
+ result = asyncio.run(exercise())
+ callback_stderr = "".join(text for text, stream in callback_chunks if stream == "stderr")
+
+ assert result.return_code == 126
+ assert callback_stderr == result.stderr
+ assert secret not in (result.stderr or "")
+ assert secret not in callback_stderr
+ assert secret not in caplog.text
+
+
+@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
+@pytest.mark.parametrize(
+ ("name", "value", "is_sensitive"),
[
- "LD_PRELOAD",
- "LD_AUDIT",
- "LD_LIBRARY_PATH",
- "DYLD_INSERT_LIBRARIES",
- "PYTHONPATH",
- "PYTHONHOME",
- "BASH_ENV",
- "ENV",
- "ZDOTDIR",
- "RUBYOPT",
- "PERL5OPT",
- "NODE_OPTIONS",
+ ("MONKEY", "banana", False),
+ ("KEYBOARD", "clacky", False),
+ ("MYSECRET", "legacy-secret-value", True),
+ # The long-value legacy fallback remains intentionally conservative.
+ ("MONKEY", "bananabanana", True),
+ ("API_KEY", "x", True),
],
)
-@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
-def test_runtime_injection_env_is_blocked_before_launcher(name: str, tmp_path: Path) -> None:
- environment = _local_environment(tmp_path)
+def test_output_redaction_balances_component_names_and_legacy_fallback(
+ tmp_path: Path,
+ name: str,
+ value: str,
+ is_sensitive: bool,
+) -> None:
+ environment = _initialized_local_environment(tmp_path)
+ callback_chunks: list[str] = []
- async def unexpected_launch(*_args: object, **_kwargs: object) -> None:
- pytest.fail("loader-controlled task environment reached the host launcher")
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
- with pytest.MonkeyPatch.context() as patch:
- patch.setattr(asyncio, "create_subprocess_exec", unexpected_launch)
- result = asyncio.run(environment.exec("true", env={name: "attacker-controlled"}))
+ async def exercise() -> object:
+ await environment.start()
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(f'printf %s "${name}"', env={name: value})
- assert result.return_code == 126
- assert name in (result.stderr or "")
+ result = asyncio.run(exercise())
+ callback_output = "".join(callback_chunks)
+
+ assert callback_output == result.stdout
+ assert (value not in callback_output) is is_sensitive
+ if not is_sensitive:
+ assert callback_output == value
@pytest.mark.skipif(os.name == "nt", reason=_NATIVE_WINDOWS_LOCAL_REASON)
@@ -3051,8 +5068,10 @@ def test_rewrite_env_values_does_not_add_shell_quotes(tmp_path: Path) -> None:
def test_local_opencode_confines_project_discovery_to_the_run_workspace(
monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
) -> None:
agent = object.__new__(SkillEvaluatorLocalOpenCode)
+ agent.logs_dir = tmp_path
agent.model_name = "nvidia/openai/gpt-oss-120b"
agent.mcp_servers = []
agent._opencode_config = {}
@@ -3362,6 +5381,490 @@ def test_harbor_preflight_system_exit_becomes_a_diagnostic(monkeypatch: pytest.M
]
+def test_singularity_prerequisite_requires_the_cli_harbor_invokes(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setattr(runner.shutil, "which", lambda executable: None if executable == "singularity" else "/bin/tool")
+
+ assert _check_prerequisites(env_mode="singularity", agents=[]) == [
+ "Harbor environment 'singularity' requires the singularity CLI on PATH."
+ ]
+
+
+def test_islo_prerequisite_requires_nonempty_host_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setenv("ISLO_API_KEY", " ")
+
+ assert _check_prerequisites(env_mode="islo", agents=[]) == [
+ "Harbor environment 'islo' requires a non-empty ISLO_API_KEY in the host environment."
+ ]
+
+
+def test_modal_custom_config_path_does_not_suppress_harbor_auth_preflight(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from harbor.environments.factory import EnvironmentFactory
+
+ from skillevaluator.tier3.harbor import runner
+
+ config_path = tmp_path / "modal.toml"
+ config_path.write_text("[default]\n", encoding="utf-8")
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setattr(runner.importlib.util, "find_spec", lambda _name: object())
+ monkeypatch.setenv("MODAL_CONFIG_PATH", str(config_path))
+ monkeypatch.setattr(
+ EnvironmentFactory,
+ "run_preflight",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(
+ SystemExit(
+ "Modal requires authentication. Run 'modal token new' to set up credentials, "
+ "or set MODAL_TOKEN_ID and MODAL_TOKEN_SECRET environment variables."
+ )
+ ),
+ )
+
+ assert _check_prerequisites(env_mode="modal", agents=[]) == [
+ "Harbor environment 'modal' is not ready: Modal requires authentication. "
+ "Run 'modal token new' to set up credentials, or set MODAL_TOKEN_ID and "
+ "MODAL_TOKEN_SECRET environment variables."
+ ]
+
+
+def test_modal_custom_config_path_must_be_an_existing_regular_file(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setenv("MODAL_CONFIG_PATH", str(tmp_path / "missing.toml"))
+
+ assert _check_prerequisites(env_mode="modal", agents=[]) == [
+ "MODAL_CONFIG_PATH must name an existing regular file."
+ ]
+
+
+def test_modal_custom_config_path_rejects_directory(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setenv("MODAL_CONFIG_PATH", str(tmp_path))
+
+ assert _check_prerequisites(env_mode="modal", agents=[]) == [
+ "MODAL_CONFIG_PATH must name an existing regular file."
+ ]
+
+
+def test_modal_custom_config_path_stat_error_is_a_bounded_diagnostic(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ class UnreadablePath:
+ def expanduser(self) -> UnreadablePath:
+ return self
+
+ def is_file(self) -> bool:
+ raise OSError("secret path detail")
+
+ monkeypatch.setenv("MODAL_CONFIG_PATH", "/private/config")
+ monkeypatch.setattr(runner, "Path", lambda _value: UnreadablePath())
+
+ assert runner._modal_custom_config_status() == (
+ False,
+ "MODAL_CONFIG_PATH must name an existing regular file.",
+ )
+
+
+def test_modal_custom_config_path_unknown_user_is_a_bounded_diagnostic(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setenv("MODAL_CONFIG_PATH", "~definitely-no-such-user-issue79/modal.toml")
+
+ assert runner._modal_custom_config_status() == (
+ False,
+ "MODAL_CONFIG_PATH must name an existing regular file.",
+ )
+
+
+@pytest.mark.parametrize("configured_path", ["", " "])
+def test_modal_custom_config_path_must_not_be_blank(
+ monkeypatch: pytest.MonkeyPatch,
+ configured_path: str,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setenv("MODAL_CONFIG_PATH", configured_path)
+
+ assert _check_prerequisites(env_mode="modal", agents=[]) == [
+ "MODAL_CONFIG_PATH must name an existing regular file."
+ ]
+
+
+def test_modal_custom_config_does_not_hide_unrelated_preflight_exit(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from harbor.environments.factory import EnvironmentFactory
+
+ from skillevaluator.tier3.harbor import runner
+
+ config_path = tmp_path / "modal.toml"
+ config_path.write_text("[default]\n", encoding="utf-8")
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setattr(runner.importlib.util, "find_spec", lambda _name: object())
+ monkeypatch.setenv("MODAL_CONFIG_PATH", str(config_path))
+ monkeypatch.setattr(
+ EnvironmentFactory,
+ "run_preflight",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(SystemExit("Modal daemon unavailable")),
+ )
+
+ assert _check_prerequisites(env_mode="modal", agents=[]) == [
+ "Harbor environment 'modal' is not ready: Modal daemon unavailable"
+ ]
+
+
+def _install_fake_kubernetes(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ kube_config_error: bool = False,
+ pod_list_error: Exception | None = None,
+) -> dict[str, object]:
+ calls: dict[str, object] = {"kube": [], "incluster": 0, "pods": [], "closed": 0}
+
+ class ConfigException(Exception):
+ pass
+
+ def load_kube_config(**kwargs: str) -> None:
+ cast_calls = calls["kube"]
+ assert isinstance(cast_calls, list)
+ cast_calls.append(dict(kwargs))
+ if kube_config_error:
+ raise ConfigException("no kube config")
+
+ def load_incluster_config() -> None:
+ calls["incluster"] = int(calls["incluster"]) + 1
+
+ class ApiClient:
+ def close(self) -> None:
+ calls["closed"] = int(calls["closed"]) + 1
+
+ class CoreV1Api:
+ def __init__(self, api_client: ApiClient) -> None:
+ assert isinstance(api_client, ApiClient)
+
+ def list_namespaced_pod(self, **kwargs: object) -> None:
+ cast_calls = calls["pods"]
+ assert isinstance(cast_calls, list)
+ cast_calls.append(dict(kwargs))
+ if pod_list_error is not None:
+ raise pod_list_error
+
+ kubernetes = SimpleNamespace(
+ client=SimpleNamespace(ApiClient=ApiClient, CoreV1Api=CoreV1Api),
+ config=SimpleNamespace(
+ ConfigException=ConfigException,
+ load_kube_config=load_kube_config,
+ load_incluster_config=load_incluster_config,
+ ),
+ )
+ monkeypatch.setitem(sys.modules, "kubernetes", kubernetes)
+ return calls
+
+
+def test_ack_cluster_readiness_uses_explicit_config_context_and_bounded_namespaced_pod_list(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ calls = _install_fake_kubernetes(monkeypatch)
+
+ runner._check_ack_cluster_readiness(
+ {"namespace": "skill-evals", "context": "production", "kubeconfig": "/config/ack"}
+ )
+
+ assert calls == {
+ "kube": [{"context": "production", "config_file": "/config/ack"}],
+ "incluster": 0,
+ "pods": [{"namespace": "skill-evals", "limit": 1, "_request_timeout": (5, 10)}],
+ "closed": 1,
+ }
+
+
+def test_ack_cluster_readiness_falls_back_to_incluster_and_closes_on_api_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ calls = _install_fake_kubernetes(
+ monkeypatch,
+ kube_config_error=True,
+ pod_list_error=RuntimeError("namespaced pod list forbidden"),
+ )
+
+ with pytest.raises(RuntimeError, match="namespaced pod list forbidden"):
+ runner._check_ack_cluster_readiness({"namespace": "skill-evals"})
+
+ assert calls == {
+ "kube": [{}],
+ "incluster": 1,
+ "pods": [{"namespace": "skill-evals", "limit": 1, "_request_timeout": (5, 10)}],
+ "closed": 1,
+ }
+
+
+def test_ack_prerequisite_performs_namespaced_pod_list_access_check(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from harbor.environments.factory import EnvironmentFactory
+
+ from skillevaluator.tier3.harbor import runner
+
+ captured: list[tuple[dict[str, object], dict[str, str]]] = []
+ subprocess_env = {"PATH": "/usr/bin", "KUBECONFIG": "/config/ack"}
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setattr(EnvironmentFactory, "run_preflight", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(
+ runner,
+ "_check_ack_cluster_readiness_subprocess",
+ lambda kwargs, *, subprocess_env: captured.append((dict(kwargs), dict(subprocess_env))),
+ )
+
+ assert (
+ _check_prerequisites(
+ env_mode="ack",
+ agents=[],
+ environment_kwargs={"namespace": "skill-evals", "context": "production"},
+ subprocess_env=subprocess_env,
+ )
+ == []
+ )
+ assert captured == [
+ (
+ {"namespace": "skill-evals", "context": "production"},
+ subprocess_env,
+ )
+ ]
+
+
+def test_ack_namespace_access_error_is_redacted(monkeypatch: pytest.MonkeyPatch) -> None:
+ from harbor.environments.factory import EnvironmentFactory
+
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setenv("HTTPS_PROXY", "http://u:pw@proxy.invalid")
+ monkeypatch.setattr(EnvironmentFactory, "run_preflight", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(
+ runner,
+ "_check_ack_cluster_readiness_subprocess",
+ lambda _kwargs, **_options: (_ for _ in ()).throw(RuntimeError("API rejected user u password pw")),
+ )
+
+ rendered = " ".join(
+ _check_prerequisites(
+ env_mode="ack",
+ agents=[],
+ environment_kwargs={"namespace": "skill-evals"},
+ )
+ )
+
+ assert " user u " not in rendered
+ assert " password pw" not in rendered
+ assert "" in rendered
+
+
+def test_ack_subprocess_probe_uses_exact_harbor_environment_and_stdin(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ captured: dict[str, object] = {}
+ child_env = {
+ "PATH": "/usr/bin",
+ "KUBECONFIG": "/config/ack",
+ "KUBERNETES_SERVICE_HOST": "kubernetes.default.svc",
+ "KUBERNETES_SERVICE_PORT": "443",
+ }
+ monkeypatch.setenv("ALIBABA_CLOUD_ACCESS_KEY_ID", "parent-only-credential")
+
+ class FakeProcess:
+ pid = 1234
+ returncode = 0
+
+ def communicate(self, input_text: str, timeout: int) -> tuple[str, str]:
+ captured["input"] = input_text
+ captured["timeout"] = timeout
+ return "", ""
+
+ def fake_popen(command: list[str], **kwargs: object) -> FakeProcess:
+ captured["command"] = list(command)
+ captured.update(kwargs)
+ return FakeProcess()
+
+ monkeypatch.setattr(runner.subprocess, "Popen", fake_popen)
+
+ runner._check_ack_cluster_readiness_subprocess(
+ {
+ "namespace": "skill-evals",
+ "context": "production",
+ "kubeconfig": "/config/ack",
+ "node_selector": {"pool": "sandbox"},
+ },
+ subprocess_env=child_env,
+ )
+
+ assert captured["env"] == child_env
+ assert "ALIBABA_CLOUD_ACCESS_KEY_ID" not in captured["env"]
+ assert json.loads(str(captured["input"])) == {
+ "namespace": "skill-evals",
+ "context": "production",
+ "kubeconfig": "/config/ack",
+ }
+ assert captured["timeout"] == runner._ACK_CLUSTER_READINESS_SUBPROCESS_TIMEOUT_SECONDS
+ assert captured["text"] is True
+ assert captured["stdin"] is subprocess.PIPE
+ assert captured["stdout"] is subprocess.PIPE
+ assert captured["stderr"] is subprocess.PIPE
+ assert captured["start_new_session"] is (os.name == "posix")
+ command = captured["command"]
+ assert isinstance(command, list)
+ assert command[:2] == [sys.executable, "-c"]
+ assert "production" not in " ".join(command)
+ assert "/config/ack" not in " ".join(command)
+
+
+def test_ack_subprocess_probe_redacts_config_values_and_child_credentials(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ context = "sensitive-context"
+ kubeconfig = "/private/ack-config"
+ child_env = {"PATH": "/usr/bin", "KUBECONFIG": kubeconfig, "HTTPS_PROXY": "http://u:pw@proxy.invalid"}
+
+ class FailedProcess:
+ pid = 1234
+ returncode = 1
+
+ def communicate(self, _input: str, timeout: int) -> tuple[str, str]:
+ assert timeout == runner._ACK_CLUSTER_READINESS_SUBPROCESS_TIMEOUT_SECONDS
+ return (
+ "",
+ f"namespaced pod list forbidden for context {context} via {kubeconfig}; proxy user u password pw",
+ )
+
+ monkeypatch.setattr(runner.subprocess, "Popen", lambda *_args, **_kwargs: FailedProcess())
+
+ with pytest.raises(RuntimeError) as exc_info:
+ runner._check_ack_cluster_readiness_subprocess(
+ {"namespace": "skill-evals", "context": context, "kubeconfig": kubeconfig},
+ subprocess_env=child_env,
+ )
+
+ rendered = str(exc_info.value)
+ assert context not in rendered
+ assert kubeconfig not in rendered
+ assert " user u " not in rendered
+ assert " password pw" not in rendered
+ assert "namespaced pod list forbidden" in rendered
+ assert "" in rendered
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
+def test_ack_subprocess_probe_timeout_kills_the_exec_auth_process_group(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ signals: list[tuple[int, signal.Signals]] = []
+
+ class HungProcess:
+ pid = 4321
+ returncode: int | None = None
+
+ def communicate(self, _input: str, timeout: int) -> tuple[str, str]:
+ raise subprocess.TimeoutExpired([sys.executable, "-c", "probe"], timeout)
+
+ def wait(self, timeout: int) -> int:
+ assert timeout == runner._ACK_CLUSTER_READINESS_REAP_TIMEOUT_SECONDS
+ self.returncode = -int(signal.SIGKILL)
+ return self.returncode
+
+ def kill(self) -> None:
+ raise AssertionError("POSIX cleanup must kill the whole process group")
+
+ monkeypatch.setattr(runner.subprocess, "Popen", lambda *_args, **_kwargs: HungProcess())
+ monkeypatch.setattr(runner.os, "killpg", lambda pid, value: signals.append((pid, value)))
+
+ with pytest.raises(RuntimeError, match="timed out after 20 seconds"):
+ runner._check_ack_cluster_readiness_subprocess(
+ {"namespace": "skill-evals"},
+ subprocess_env={"PATH": "/usr/bin", "KUBECONFIG": "/config/ack"},
+ )
+
+ assert signals == [(4321, signal.SIGKILL)]
+
+
+def test_docker_prerequisite_redacts_proxy_uri_and_short_detached_credentials(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setenv("HTTPS_PROXY", "http://u:pw@proxy.invalid")
+ monkeypatch.setattr(
+ runner.subprocess,
+ "run",
+ lambda *_args, **_kwargs: subprocess.CompletedProcess(
+ [],
+ 1,
+ stdout="",
+ stderr="proxy http://u:pw@proxy.invalid rejected user u password pw",
+ ),
+ )
+
+ rendered = " ".join(_check_prerequisites(env_mode="docker", agents=[]))
+
+ assert "http://u:pw@" not in rendered
+ assert " user u " not in rendered
+ assert " password pw" not in rendered
+ assert "" in rendered
+
+
+def test_harbor_preflight_exception_redacts_detached_proxy_credentials(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from harbor.environments.factory import EnvironmentFactory
+
+ from skillevaluator.tier3.harbor import runner
+
+ monkeypatch.setattr(runner, "_harbor_bin", lambda: "/fake/harbor")
+ monkeypatch.setenv("HTTPS_PROXY", "http://u:pw@proxy.invalid")
+ monkeypatch.setattr(
+ EnvironmentFactory,
+ "run_preflight",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("proxy rejected user u password pw")),
+ )
+
+ rendered = " ".join(_check_prerequisites(env_mode="e2b", agents=[]))
+
+ assert " user u " not in rendered
+ assert " password pw" not in rendered
+ assert "" in rendered
+
+
def test_harbor_preflight_does_not_swallow_keyboard_interrupt(monkeypatch: pytest.MonkeyPatch) -> None:
from harbor.environments.factory import EnvironmentFactory
diff --git a/tests/test_harbor_metrics_truth.py b/tests/test_harbor_metrics_truth.py
index eaf33479..1098acdd 100644
--- a/tests/test_harbor_metrics_truth.py
+++ b/tests/test_harbor_metrics_truth.py
@@ -3,6 +3,7 @@
from __future__ import annotations
+import json
import math
import os
import subprocess
@@ -12,6 +13,7 @@
import pytest
+from skillevaluator.tier3 import results_location
from skillevaluator.tier3.harbor import collector as collector_module
from skillevaluator.tier3.harbor.collector import (
_compute_lift,
@@ -24,26 +26,155 @@
_pass_rate_delta,
_pass_summary,
_probability_text,
+ _public_pass_summary,
_wilson_score_interval,
)
from skillevaluator.tier3.harbor.metrics import (
CUSTOM_ONLY_METRIC_SET,
DEFAULT_METRIC_SET,
DEFAULT_METRICS,
+ LEGACY_METRIC_SET,
+ LEGACY_METRICS,
+ MAX_CUSTOM_METRIC_NAME_BYTES,
+ MAX_CUSTOM_METRICS,
+ CustomMetricContractError,
+ average_custom_metrics,
average_metrics,
+ custom_metric_contract_error,
extract_custom_metrics,
+ metric_set_for_reward,
metric_value,
overall_score,
+ rewards_have_mixed_metric_contracts,
)
-@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), float("-inf")])
-def test_metric_inputs_reject_nonfinite_values(invalid: float) -> None:
+def test_custom_metric_names_filter_credentials_but_keep_explicit_metric_terms() -> None:
+ credentials = [
+ "sk-abcdefghijk",
+ "ghp_" + ("a" * 36),
+ "gho_" + ("a" * 36),
+ "ghu_" + ("a" * 36),
+ "ghs_" + ("a" * 36),
+ "ghr_" + ("a" * 36),
+ "qualityghp_" + ("a" * 36),
+ "qualitygho_" + ("a" * 36) + "suffix",
+ "ghs_123456789_" + ("a" * 32) + "." + ("b" * 32) + "." + ("c" * 32),
+ "github_pat_" + ("a" * 30),
+ "".join(("xoxb-", "1234567890-abcdefghijklmnopqrstuvwx")), # noqa: FLY002
+ "AIza" + ("A" * 35),
+ "glpat-" + ("a" * 20),
+ ]
+ reward = {
+ "custom_metrics": {
+ **dict.fromkeys(credentials, 0.1),
+ **dict.fromkeys((f"quality_{credential}" for credential in credentials), 0.1),
+ "quality_nvapi-abcdefghijk": 0.1,
+ "quality_crsr_0123456789abcdef": 0.1,
+ "api_key_quality": 0.2,
+ "quality": 0.3,
+ "secret_handling": 0.4,
+ "token_efficiency": 0.5,
+ }
+ }
+
+ assert custom_metric_contract_error(reward) is None
+ assert extract_custom_metrics(reward) == {
+ "quality": 0.3,
+ "secret_handling": 0.4,
+ "token_efficiency": 0.5,
+ }
+
+
+@pytest.mark.parametrize(
+ "exact",
+ [
+ "x" * MAX_CUSTOM_METRIC_NAME_BYTES,
+ "é" * (MAX_CUSTOM_METRIC_NAME_BYTES // len("é".encode())),
+ ],
+ ids=("ascii", "multibyte"),
+)
+def test_custom_metric_contract_enforces_name_byte_boundary_without_aliasing(exact: str) -> None:
+ oversized = exact + "x"
+
+ assert custom_metric_contract_error({"custom_metrics": {exact: 1.0}}) is None
+ assert extract_custom_metrics({"custom_metrics": {exact: 1.0}}) == {exact: 1.0}
+ assert custom_metric_contract_error({"custom_metrics": {oversized: 1.0}}) is not None
+ assert extract_custom_metrics({"custom_metrics": {oversized: 1.0}}) == {}
+
+
+def test_custom_metric_contract_enforces_per_reward_and_union_cardinality() -> None:
+ exact = {f"metric_{index:03d}": 1.0 for index in range(MAX_CUSTOM_METRICS)}
+ oversized = {**exact, "one_too_many": 1.0}
+ exact_plus_unsafe = {**exact, "quality_sk-abcdefghijk": 1.0}
+
+ assert custom_metric_contract_error({"custom_metrics": exact}) is None
+ assert len(extract_custom_metrics({"custom_metrics": exact})) == MAX_CUSTOM_METRICS
+ assert custom_metric_contract_error({"custom_metrics": exact_plus_unsafe}) is None
+ assert len(extract_custom_metrics({"custom_metrics": exact_plus_unsafe})) == MAX_CUSTOM_METRICS
+ assert custom_metric_contract_error({"custom_metrics": oversized}) is not None
+ with pytest.raises(CustomMetricContractError, match="per reward"):
+ average_custom_metrics([{"custom_metrics": oversized}])
+
+ left = {f"left_{index:03d}": 1.0 for index in range(MAX_CUSTOM_METRICS // 2 + 1)}
+ right = {f"right_{index:03d}": 1.0 for index in range(MAX_CUSTOM_METRICS // 2 + 1)}
+ with pytest.raises(CustomMetricContractError, match="per condition"):
+ average_custom_metrics([{"custom_metrics": left}, {"custom_metrics": right}])
+
+
+def test_explicit_custom_metrics_reject_reserved_name_collisions() -> None:
+ reward = {"custom_metrics": {"security": 0.0, "quality": 1.0}, "overall": 1.0}
+
+ assert "collides" in (custom_metric_contract_error(reward) or "")
+ with pytest.raises(CustomMetricContractError, match="collides"):
+ average_custom_metrics([reward])
+
+
+@pytest.mark.parametrize("malformed", [None, 0.5, "quality", [0.5]])
+def test_custom_metrics_container_must_be_an_object(malformed: object) -> None:
+ reward = {"custom_metrics": malformed, "quality": 0.8, "overall": 0.8}
+
+ assert "container" in (custom_metric_contract_error(reward) or "")
+ assert extract_custom_metrics(reward) == {"quality": 0.8}
+ with pytest.raises(CustomMetricContractError, match="container"):
+ average_custom_metrics([reward])
+
+
+def test_custom_metric_extraction_unions_explicit_nested_and_top_level_dict_scores() -> None:
+ reward = {
+ "custom_metrics": {"quality": 0.8},
+ "metrics": {"coverage": {"score": 0.7}},
+ "domain_score": {"score": 0.6},
+ "api_key_quality": {"score": 0.9},
+ }
+
+ assert custom_metric_contract_error(reward) is None
+ assert extract_custom_metrics(reward) == {
+ "quality": 0.8,
+ "coverage": 0.7,
+ "domain_score": 0.6,
+ }
+
+
+@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), float("-inf"), 10**400])
+def test_metric_inputs_reject_nonfinite_values(invalid: float | int) -> None:
assert metric_value({"security": invalid}, "security") is None
assert metric_value({"metrics": {"security": {"score": invalid}}}, "security") is None
assert extract_custom_metrics({"custom_metrics": {"latency": invalid}}) == {}
+@pytest.mark.parametrize("invalid", [-0.01, 1.01, 1e308])
+def test_metric_inputs_reject_scores_outside_the_documented_unit_interval(invalid: float) -> None:
+ reward = {"metric_set": DEFAULT_METRIC_SET, **dict.fromkeys(DEFAULT_METRICS, invalid)}
+
+ assert metric_value({"security": invalid}, "security") is None
+ assert metric_value({"metrics": {"security": {"score": invalid}}}, "security") is None
+ assert extract_custom_metrics({"custom_metrics": {"latency": invalid}}) == {}
+ assert overall_score(reward) is None
+ assert overall_score({"metric_set": CUSTOM_ONLY_METRIC_SET, "overall": invalid}) is None
+ assert overall_score({**dict.fromkeys(DEFAULT_METRICS, invalid), "overall": 0.8}) is None
+
+
def test_average_metrics_omits_unavailable_canonical_metrics() -> None:
scores, metric_set, metrics = average_metrics(
[
@@ -60,6 +191,44 @@ def test_average_metrics_omits_unavailable_canonical_metrics() -> None:
assert scores == {"security": 1.0}
+def test_explicit_custom_metric_set_cannot_spoof_canonical_metrics() -> None:
+ custom = {
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 0.0),
+ "overall": 0.25,
+ }
+ standard = {
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "overall": 1.0,
+ }
+
+ assert metric_set_for_reward(custom) == (CUSTOM_ONLY_METRIC_SET, ())
+ assert overall_score(custom) == pytest.approx(0.25)
+ scores, metric_set, metrics = average_metrics([custom, standard])
+ assert metric_set == DEFAULT_METRIC_SET
+ assert metrics == DEFAULT_METRICS
+ assert scores == dict.fromkeys(DEFAULT_METRICS, 1.0)
+
+
+@pytest.mark.parametrize(
+ "rewards",
+ [
+ [
+ {"metric_set": DEFAULT_METRIC_SET, **dict.fromkeys(DEFAULT_METRICS, 1.0)},
+ {"metric_set": LEGACY_METRIC_SET, **dict.fromkeys(LEGACY_METRICS, 1.0)},
+ ],
+ [
+ {"metric_set": "domain-grader-v1", "overall": 1.0},
+ {"metric_set": "domain-grader-v2", "overall": 1.0},
+ ],
+ ],
+ ids=("default-v1-and-v2", "distinct-custom-contracts"),
+)
+def test_distinct_reward_metric_set_contracts_are_mixed(rewards: list[dict[str, object]]) -> None:
+ assert rewards_have_mixed_metric_contracts(rewards) is True
+
+
def test_overall_score_requires_a_complete_finite_metric_set() -> None:
complete = dict.fromkeys(DEFAULT_METRICS, 0.8)
incomplete = dict(complete)
@@ -117,6 +286,100 @@ def test_pass_summary_marks_incomplete_reward_unscored() -> None:
assert math.isfinite(summary["rate"])
+def test_large_case_sets_publish_bounded_samples_without_losing_exact_pairing() -> None:
+ case_ids = [f"case-{index:05d}-{'x' * 80}" for index in range(20_000)]
+ with_rewards = [
+ {
+ "entry_id": case_id,
+ "_trial_name": f"trial-{index:05d}",
+ "_trial_root_name": f"trial-{index:05d}",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 1.0,
+ }
+ for index, case_id in enumerate(case_ids)
+ ]
+ without_rewards = [{**reward, "overall": 0.0} for reward in with_rewards]
+
+ with_summary = _pass_summary(
+ with_rewards,
+ n_attempts=1,
+ pass_threshold=0.5,
+ expected_cases=len(case_ids),
+ expected_case_ids=case_ids,
+ )
+ without_summary = _pass_summary(
+ without_rewards,
+ n_attempts=1,
+ pass_threshold=0.5,
+ expected_cases=len(case_ids),
+ expected_case_ids=case_ids,
+ )
+ paired = _paired_pass_comparison(with_summary, without_summary)
+ public = _public_pass_summary(with_summary)
+
+ assert with_summary["passed_cases"] == len(case_ids)
+ assert paired["pairing_status"] == "complete"
+ assert paired["paired_cases"] == len(case_ids)
+ assert paired["with_skill_only_pass"] == len(case_ids)
+ assert public["case_details_total"] == len(case_ids)
+ assert public["case_details_shown"] == collector_module.PUBLISHED_CASE_DETAILS_MAX
+ assert public["case_details_truncated"] is True
+ assert "_pairing_cases" not in public
+ assert len(json.dumps(public, separators=(",", ":")).encode()) < collector_module.GENERATED_JSON_MAX_BYTES
+ assert results_location._legacy_pass_at_k_is_complete(
+ public,
+ num_trials=len(case_ids),
+ require_scored_attempt=True,
+ expected_scored_attempts=len(case_ids),
+ )
+
+
+def test_large_missing_case_set_has_bounded_exact_execution_diagnostics() -> None:
+ case_ids = [f"case-{index:05d}-{'x' * 80}" for index in range(20_000)]
+
+ summary = _condition_execution_summary(
+ [],
+ expected_case_ids=case_ids,
+ expected_cases=len(case_ids),
+ n_attempts=1,
+ job_failure="job did not produce trials",
+ )
+ encoded = json.dumps(summary, separators=(",", ":")).encode()
+
+ assert summary["execution_status"] == "failed"
+ assert summary["expected_attempts"] == len(case_ids)
+ assert summary["scored_attempts"] == 0
+ assert any("showing 32 of 20000" in error for error in summary["execution_errors"])
+ assert len(encoded) < collector_module.GENERATED_JSON_MAX_BYTES
+
+
+@pytest.mark.parametrize("failure_kind", ["runtime_failures", "reward_failures"])
+def test_large_failure_sets_publish_bounded_samples_with_exact_counts(failure_kind: str) -> None:
+ failures = [{"trial": f"trial-{index:05d}", "reason": "upstream failure " + ("x" * 600)} for index in range(4_096)]
+
+ summary = _condition_execution_summary(
+ [],
+ expected_case_ids=[],
+ expected_cases=0,
+ n_attempts=1,
+ job_failure="",
+ **{failure_kind: failures},
+ )
+ encoded = json.dumps(summary, separators=(",", ":")).encode()
+ prefix = "runtime_failure_details" if failure_kind == "runtime_failures" else "reward_failure_details"
+
+ assert summary["execution_status"] == "failed"
+ assert summary[f"{prefix}_total"] == len(failures)
+ assert summary[f"{prefix}_shown"] == collector_module.PUBLISHED_FAILURE_DETAILS_MAX
+ assert summary[f"{prefix}_truncated"] is True
+ assert summary["execution_error_details_truncated"] is False
+ assert any(
+ f"showing {collector_module.PUBLISHED_FAILURE_DETAILS_MAX} of 4096" in error
+ for error in summary["execution_errors"]
+ )
+ assert len(encoded) < collector_module.GENERATED_JSON_MAX_BYTES
+
+
@pytest.mark.parametrize(
("successes", "total", "expected"),
[
diff --git a/tests/test_harbor_negative_control_evidence.py b/tests/test_harbor_negative_control_evidence.py
index 32d8bbb5..b5a48102 100644
--- a/tests/test_harbor_negative_control_evidence.py
+++ b/tests/test_harbor_negative_control_evidence.py
@@ -14,6 +14,7 @@
from skillevaluator.tier3.harbor.collector import (
REWARD_DIAGNOSTIC_STRING_MAX_CHARS,
+ _trajectory_skill_invoked,
collect_harbor_results,
)
@@ -45,6 +46,7 @@ def _trajectory(*, skill: str | None = None) -> dict[str, Any]:
)
return {
"schema_version": "ATIF-v1.2",
+ "agent": {"name": "opencode", "version": "test"},
"steps": [
{
"step_id": 1,
@@ -118,6 +120,22 @@ def _persisted_reward(tmp_path: Path, variant: str, trial_name: str) -> dict[str
return json.loads(path.read_text(encoding="utf-8"))
+def test_optional_no_tool_call_agent_step_does_not_hide_later_skill_invocation() -> None:
+ trajectory = _trajectory(skill="demo")
+ trajectory["schema_version"] = "ATIF-v1.7"
+ trajectory["steps"].insert(
+ 0,
+ {
+ "step_id": 1,
+ "source": "agent",
+ "message": "I will inspect the skill next.",
+ },
+ )
+ trajectory["steps"][1]["step_id"] = 2
+
+ assert _trajectory_skill_invoked(trajectory, "demo") is True
+
+
def test_collect_persists_target_invocation_for_both_single_step_variants(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
for variant, observed_skill in (("with", "demo"), ("without", "unrelated")):
@@ -159,7 +177,89 @@ def test_collect_persists_target_invocation_for_both_single_step_variants(tmp_pa
assert {path.name for path in trial_dir.iterdir()} == {"result.json", "reward.json", "trajectory.json"}
-def test_collect_redacts_successful_standard_reward_without_diagnostic_truncation(tmp_path: Path) -> None:
+def test_collect_materializes_single_step_continuation_for_evidence_and_output(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_name = "case-with_attempt001"
+ root = _trajectory(skill="unrelated")
+ root["session_id"] = "root-session"
+ root["trajectory_id"] = "root-trajectory"
+ root["continued_trajectory_ref"] = "trajectory.cont-1.json"
+ continuation = _trajectory(skill="demo")
+ continuation["session_id"] = "continuation-session"
+ continuation["trajectory_id"] = "continuation-trajectory"
+ trial_dir = _write_trial(
+ jobs_dir,
+ variant="with",
+ trial_name=trial_name,
+ reward=dict(STANDARD_REWARD),
+ trajectory=root,
+ )
+ (trial_dir / "agent" / "trajectory.cont-1.json").write_text(json.dumps(continuation), encoding="utf-8")
+ (trial_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "case-with",
+ "verifier_result": {"rewards": dict(STANDARD_REWARD)},
+ "step_results": None,
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_job_result(jobs_dir / "demo-opencode-with", [trial_name])
+
+ _collect(tmp_path)
+
+ persisted_reward = _persisted_reward(tmp_path, "with", trial_name)
+ assert persisted_reward["skill_invoked"] is True
+ assert persisted_reward["invocation_evidence_source"] == "trajectory"
+ trajectory_path = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "trajectory.json"
+ persisted_trajectory = json.loads(trajectory_path.read_text(encoding="utf-8"))
+ assert "continued_trajectory_ref" not in persisted_trajectory
+ assert len(persisted_trajectory["steps"]) == 2
+ assert persisted_trajectory["trajectory_id"].startswith("skillevaluator-continuation-")
+
+
+def test_collect_rejects_contradictory_root_and_authoritative_step_trajectories(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_name = "contradictory-topology_attempt001"
+ trial_dir = _write_trial(
+ jobs_dir,
+ variant="with",
+ trial_name=trial_name,
+ reward=dict(STANDARD_REWARD),
+ trajectory=_trajectory(skill="demo"),
+ )
+ step_agent_dir = trial_dir / "steps" / "only" / "agent"
+ step_agent_dir.mkdir(parents=True)
+ (step_agent_dir / "trajectory.json").write_text(json.dumps(_trajectory(skill="demo")), encoding="utf-8")
+ (trial_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "contradictory-topology",
+ "verifier_result": {"rewards": dict(STANDARD_REWARD)},
+ "step_results": [{"step_name": "only"}],
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_job_result(jobs_dir / "demo-opencode-with", [trial_name])
+
+ _collect(tmp_path)
+
+ persisted_reward = _persisted_reward(tmp_path, "with", trial_name)
+ assert "skill_invoked" not in persisted_reward
+ trial_out = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name
+ assert not (trial_out / "trajectory.json").exists()
+ manifest = json.loads((trial_out / "artifact_manifest.json").read_text(encoding="utf-8"))
+ assert {
+ "name": "trajectory.json",
+ "reason": "contradictory_root_and_multi_step_trajectories",
+ } in manifest["skipped"]
+
+
+def test_collect_omits_rejected_metric_names_without_diagnostic_truncation(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
trial_name = "successful-redaction_attempt001"
long_note = "x" * (REWARD_DIAGNOSTIC_STRING_MAX_CHARS + 1)
@@ -202,17 +302,17 @@ def test_collect_redacts_successful_standard_reward_without_diagnostic_truncatio
persisted = _persisted_reward(tmp_path, "with", trial_name)
assert persisted["skill_invoked"] is True
assert persisted["invocation_evidence_source"] == "trajectory"
- assert persisted["details"]["api_key"] == ""
+ assert "api_key" not in persisted["details"]
assert "secret-token-value" not in persisted["details"]["message"]
assert "" in persisted["details"]["message"]
assert persisted["details"]["safe_note"] == long_note
- assert persisted["details"]["password"] == ""
+ assert "password" not in persisted["details"]
assert persisted["custom_metrics"]["secret_safety"] == 0.91
for name in ("api_key", "api_token", "private_token", "client_token"):
- assert persisted["custom_metrics"][name] == ""
+ assert name not in persisted["custom_metrics"]
for name in ("password", "github_token", "gitlab_token", "ssh_key", "signing_key"):
- assert persisted["metrics"][name] == ""
- assert persisted["api_key"] == ""
+ assert name not in persisted["metrics"]
+ assert "api_key" not in persisted
def test_collect_persists_native_skill_tool_invocation(tmp_path: Path) -> None:
@@ -624,14 +724,23 @@ def test_collect_omits_invocation_evidence_when_shell_parser_hits_a_bound(tmp_pa
),
(
{
+ "schema_version": "ATIF-v1.7",
+ "agent": {"name": "opencode", "version": "test"},
"steps": [
- {"source": "user", "message": "request"},
- {"source": "agent", "message": "done", "tool_calls": []},
- ]
+ {"step_id": 1, "source": "user", "message": "request"},
+ {"step_id": 2, "source": "agent", "message": "done", "tool_calls": []},
+ ],
+ },
+ False,
+ ),
+ (
+ {
+ "schema_version": "ATIF-v1.7",
+ "agent": {"name": "opencode", "version": "test"},
+ "steps": [{"step_id": 1, "source": "agent", "message": "done", "tool_calls": []}],
},
False,
),
- ({"steps": [{"source": "agent", "message": "done", "tool_calls": []}]}, False),
],
ids=(
"empty",
@@ -726,18 +835,12 @@ def test_collect_ignores_non_agent_skill_tool_calls(tmp_path: Path) -> None:
trial_name = "non-agent-skill-call_attempt001"
trajectory = {
"schema_version": "ATIF-v1.2",
+ "agent": {"name": "opencode", "version": "test"},
"steps": [
{
"step_id": 1,
"source": "user",
- "message": "Use demo",
- "tool_calls": [
- {
- "tool_call_id": "call-user",
- "function_name": "Skill",
- "arguments": {"skill": "demo"},
- }
- ],
+ "message": 'Untrusted text: Skill({"skill": "demo"})',
},
{
"step_id": 2,
@@ -763,14 +866,94 @@ def test_collect_ignores_non_agent_skill_tool_calls(tmp_path: Path) -> None:
assert persisted["invocation_evidence_source"] == "trajectory"
+@pytest.mark.parametrize(("include_current_step", "expected"), [(False, None), (True, False)])
+def test_collect_ignores_copied_context_skill_invocation(
+ tmp_path: Path,
+ include_current_step: bool,
+ expected: bool | None,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_name = "copied-context_attempt001"
+ trajectory = _trajectory(skill="demo")
+ trajectory["steps"][0]["is_copied_context"] = True
+ if include_current_step:
+ trajectory["steps"].append(
+ {
+ "step_id": 2,
+ "source": "agent",
+ "message": "current execution",
+ "tool_calls": [],
+ "observation": {"results": []},
+ }
+ )
+ _write_trial(
+ jobs_dir,
+ variant="with",
+ trial_name=trial_name,
+ reward=dict(STANDARD_REWARD),
+ trajectory=trajectory,
+ )
+ _write_job_result(jobs_dir / "demo-opencode-with", [trial_name])
+
+ _collect(tmp_path)
+
+ persisted = _persisted_reward(tmp_path, "with", trial_name)
+ if expected is None:
+ assert "skill_invoked" not in persisted
+ assert "invocation_evidence_source" not in persisted
+ else:
+ assert persisted["skill_invoked"] is expected
+ assert persisted["invocation_evidence_source"] == "trajectory"
+
+
+@pytest.mark.parametrize(("referenced", "expected"), [(True, True), (False, False)])
+def test_collect_counts_only_referenced_subagent_invocation(
+ tmp_path: Path,
+ referenced: bool,
+ expected: bool,
+) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_name = "subagent-evidence_attempt001"
+ root = _trajectory()
+ root["schema_version"] = "ATIF-v1.7"
+ root["trajectory_id"] = "root"
+ child = _trajectory(skill="demo")
+ child["schema_version"] = "ATIF-v1.7"
+ child["trajectory_id"] = "child"
+ root["subagent_trajectories"] = [child]
+ if referenced:
+ root["steps"][0]["observation"] = {
+ "results": [
+ {
+ "content": "delegated",
+ "subagent_trajectory_ref": [{"trajectory_id": "child"}],
+ }
+ ]
+ }
+ _write_trial(
+ jobs_dir,
+ variant="with",
+ trial_name=trial_name,
+ reward=dict(STANDARD_REWARD),
+ trajectory=root,
+ )
+ _write_job_result(jobs_dir / "demo-opencode-with", [trial_name])
+
+ _collect(tmp_path)
+
+ persisted = _persisted_reward(tmp_path, "with", trial_name)
+ assert persisted["skill_invoked"] is expected
+ assert persisted["invocation_evidence_source"] == "trajectory"
+
+
def test_collect_leaves_custom_only_reward_and_artifacts_unchanged(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
trial_name = "custom_attempt001"
custom_reward = {
"entry_id": "custom",
"overall": 0.75,
- "custom_metrics": {"skill_execution": 0.25, "quality": 0.8},
- "details": {"skill_execution": {"note": "custom data"}},
+ "custom_metrics": {"domain_score": 0.25, "quality": 0.8},
+ "details": {"domain_score": {"note": "custom data"}},
"skill_invoked": "verifier-owned",
"routing_passed": "verifier-owned",
"invocation_evidence_source": "custom-verifier",
@@ -791,7 +974,7 @@ def test_collect_leaves_custom_only_reward_and_artifacts_unchanged(tmp_path: Pat
assert persisted[key] == custom_reward[key]
agent = result["agents"]["opencode"]
assert agent["with_skill"] == {}
- assert agent["custom_with_skill"] == {"quality": 0.8}
+ assert agent["custom_with_skill"] == {"domain_score": 0.25, "quality": 0.8}
summary = json.loads(
(tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
)
@@ -800,25 +983,15 @@ def test_collect_leaves_custom_only_reward_and_artifacts_unchanged(tmp_path: Pat
assert {path.name for path in trial_dir.iterdir()} == {"reward.json", "trajectory.json"}
-@pytest.mark.parametrize(
- "custom_score_fields",
- [
- {"skill_execution": 0.25, "quality": 0.8},
- {"custom_metrics": {"skill_execution": 0.25, "quality": 0.8}},
- ],
- ids=("top-level", "nested"),
-)
-def test_collect_respects_explicit_custom_metric_set_with_standard_named_metric(
- tmp_path: Path,
- custom_score_fields: dict[str, Any],
-) -> None:
+def test_collect_respects_custom_only_top_level_standard_named_metric(tmp_path: Path) -> None:
jobs_dir = tmp_path / "jobs"
trial_name = "explicit_custom_attempt001"
custom_reward = {
"entry_id": "custom",
"metric_set": "custom-only",
"overall": 0.75,
- **custom_score_fields,
+ "skill_execution": 0.25,
+ "quality": 0.8,
"skill_invoked": "custom-verifier-value",
"routing_passed": "custom-verifier-value",
"invocation_evidence_source": "custom-verifier",
@@ -837,3 +1010,30 @@ def test_collect_respects_explicit_custom_metric_set_with_standard_named_metric(
persisted = _persisted_reward(tmp_path, "with", trial_name)
for key, expected in custom_reward.items():
assert persisted[key] == expected
+
+
+def test_collect_rejects_reserved_name_inside_explicit_custom_metrics(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ trial_name = "explicit_custom_attempt001"
+ custom_reward = {
+ "entry_id": "custom",
+ "metric_set": "custom-only",
+ "overall": 0.75,
+ "custom_metrics": {"skill_execution": 0.25, "quality": 0.8},
+ }
+ _write_trial(
+ jobs_dir,
+ variant="with",
+ trial_name=trial_name,
+ reward=custom_reward,
+ trajectory=_trajectory(),
+ )
+ _write_job_result(jobs_dir / "demo-opencode-with", [trial_name])
+
+ result = _collect(tmp_path)
+
+ persisted = _persisted_reward(tmp_path, "with", trial_name)
+ assert result["execution_status"] == "failed"
+ assert result["agents"]["opencode"]["conditions"]["with_skill"]["scored_attempts"] == 0
+ assert persisted["evaluation_status"] == "failed"
+ assert "custom metric" in json.dumps(persisted["evaluation_errors"]).casefold()
diff --git a/tests/test_harbor_runner_artifacts.py b/tests/test_harbor_runner_artifacts.py
index 3befe35b..6c8786cb 100644
--- a/tests/test_harbor_runner_artifacts.py
+++ b/tests/test_harbor_runner_artifacts.py
@@ -3,11 +3,30 @@
from __future__ import annotations
+import hashlib
import json
+import os
from pathlib import Path
+from typing import Any
+
+import pytest
import skillevaluator.tier3.harbor.runner as harbor_runner
from skillevaluator import __version__
+from skillevaluator.tier3.harbor import collector as harbor_collector
+from skillevaluator.tier3.harbor import report_data
+
+_SNAPSHOT_LIMIT_ERROR = (
+ "Dataset snapshot exceeds the 2 MiB, depth-64, or 50,000-node publication limit; "
+ "reduce dataset size or structural complexity."
+)
+
+
+def _write_staged_entry(run_dir: Path, task: str, entry: dict[str, object]) -> Path:
+ entry_path = run_dir / "_harbor-tasks" / task / "tests" / "entry.json"
+ entry_path.parent.mkdir(parents=True)
+ entry_path.write_text(json.dumps(entry, separators=(",", ":")), encoding="utf-8")
+ return entry_path
def test_runner_persists_deduplicated_dataset_truth(tmp_path: Path) -> None:
@@ -33,3 +52,689 @@ def test_runner_persists_deduplicated_dataset_truth(tmp_path: Path) -> None:
}
assert snapshot["dataset_digest"].startswith("sha256:")
assert json.loads((run_dir / "dataset_snapshot.json").read_text(encoding="utf-8")) == snapshot
+
+
+def test_runner_rejects_snapshot_whose_combined_entries_exceed_publication_bytes(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ _write_staged_entry(run_dir, "first", {"id": "first", "prompt": "a" * 1_100_000})
+ _write_staged_entry(run_dir, "second", {"id": "second", "prompt": "b" * 1_100_000})
+
+ with pytest.raises(ValueError, match=rf"^{_SNAPSHOT_LIMIT_ERROR}$"):
+ harbor_runner._persist_dataset_truth(run_dir, fallback_task_ids=[])
+
+ assert not (run_dir / "dataset_snapshot.json").exists()
+ assert not list(run_dir.glob(".dataset_snapshot.*"))
+
+
+def test_runner_rejects_snapshot_when_wrapper_pushes_entry_over_node_limit(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ _write_staged_entry(run_dir, "case", {"id": "case", "values": [0] * 49_990})
+
+ with pytest.raises(ValueError, match=rf"^{_SNAPSHOT_LIMIT_ERROR}$"):
+ harbor_runner._persist_dataset_truth(run_dir, fallback_task_ids=[])
+
+ assert not (run_dir / "dataset_snapshot.json").exists()
+
+
+def test_runner_rejects_snapshot_when_wrapper_pushes_entry_over_depth_limit(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ nested: object = "leaf"
+ for _ in range(62):
+ nested = {"value": nested}
+ _write_staged_entry(run_dir, "case", {"id": "case", "metadata": nested})
+
+ with pytest.raises(ValueError, match=rf"^{_SNAPSHOT_LIMIT_ERROR}$"):
+ harbor_runner._persist_dataset_truth(run_dir, fallback_task_ids=[])
+
+ assert not (run_dir / "dataset_snapshot.json").exists()
+
+
+def test_runner_round_trips_near_byte_limit_snapshot_and_embeds_only_manifest(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ _write_staged_entry(run_dir, "case", {"id": "case", "prompt": "p" * 1_900_000})
+
+ snapshot = harbor_runner._persist_dataset_truth(run_dir, fallback_task_ids=[])
+ persisted = run_dir / "dataset_snapshot.json"
+ manifest = report_data.dataset_snapshot_manifest(snapshot)
+
+ assert persisted.stat().st_size <= report_data.DATASET_SNAPSHOT_MAX_BYTES
+ assert report_data.load_dataset_snapshot(run_dir) == snapshot
+ assert "dataset" not in manifest
+ assert manifest == {
+ "schema_version": snapshot["schema_version"],
+ "evaluator_version": snapshot["evaluator_version"],
+ "dataset_summary": snapshot["dataset_summary"],
+ "dataset_digest": snapshot["dataset_digest"],
+ "dataset_digest_algorithm": snapshot["dataset_digest_algorithm"],
+ }
+
+
+def _write_json(path: Path, value: object) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(value, indent=2), encoding="utf-8")
+
+
+def _max_detail_pass_summary() -> dict[str, Any]:
+ cases: dict[str, Any] = {}
+ for index in range(256):
+ case_id = f"case-{index:03d}-" + ("c" * 500)
+ cases[case_id] = {
+ "passed": True,
+ "first_pass_attempt": 1,
+ "attempts_used": 2,
+ "attempts_skipped": 0,
+ "attempts_missing": 0,
+ "best_score": 1.0,
+ "attempts": [
+ {
+ "attempt": attempt,
+ "trial": f"trial-{index:03d}-{attempt}-" + ("t" * 490),
+ "score": 1.0,
+ "passed": True,
+ }
+ for attempt in (1, 2)
+ ],
+ "attempt_details_total": 2,
+ "attempt_details_shown": 2,
+ "attempt_details_truncated": False,
+ }
+ return {
+ "k": 2,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "passed_cases": 256,
+ "failed_cases": 0,
+ "total_cases": 256,
+ "rate": 1.0,
+ "rate_interval": {"lower": 0.98, "upper": 1.0},
+ "attempts_used": 512,
+ "max_attempts_possible": 512,
+ "avg_attempts_used": 2.0,
+ "extra_case_count": 0,
+ "extra_cases": [],
+ "extra_cases_truncated": False,
+ "case_details_total": 256,
+ "case_details_shown": 256,
+ "case_details_truncated": False,
+ "case_details_limit": 256,
+ "cases": cases,
+ }
+
+
+def _max_detail_result(run_dir: Path) -> dict[str, Any]:
+ agents: dict[str, Any] = {}
+ pass_summary = _max_detail_pass_summary()
+ metric_names = [f"metric-{index:03d}-" + ("m" * 244) for index in range(128)]
+ custom_scores = dict.fromkeys(metric_names, 0.75)
+ custom_lift = {name: {"with_skill": 0.75, "without_skill": 0.25, "delta": 0.5} for name in metric_names}
+ failures = [
+ {"trial": f"trial-{index:03d}", "reason": f"failure-{index:03d}: " + ("r" * 2_000)} for index in range(32)
+ ]
+ condition_errors = [failure["reason"] for failure in failures]
+ security_cases = {
+ f"case-{index:03d}-" + ("s" * 500): {
+ "status": "with_skill_unsafe",
+ "with_skill_findings": 1,
+ "baseline_findings": 0,
+ }
+ for index in range(256)
+ }
+ security_attribution = {
+ "likely_skill_related": 256,
+ "likely_baseline_prompt_or_environment": 0,
+ "skill_may_have_improved_safety": 0,
+ "ambiguous_with_skill_only": 0,
+ "unknown_no_baseline": 0,
+ "case_details_total": 256,
+ "case_details_shown": 256,
+ "case_details_truncated": False,
+ "case_details_limit": 256,
+ "cases": security_cases,
+ }
+ condition = {
+ "execution_status": "failed",
+ "execution_errors": condition_errors,
+ "execution_error_details_total": 32,
+ "execution_error_details_shown": 32,
+ "execution_error_details_truncated": False,
+ "expected_attempts": 512,
+ "scored_attempts": 480,
+ "runtime_failure_details_total": 32,
+ "runtime_failure_details_shown": 32,
+ "runtime_failure_details_truncated": False,
+ "reward_failure_details_total": 32,
+ "reward_failure_details_shown": 32,
+ "reward_failure_details_truncated": False,
+ }
+ pass_lift = {
+ "with_skill": 1.0,
+ "without_skill": 1.0,
+ "delta": 0.0,
+ "passed_cases_delta": 0,
+ }
+ run_config_agents: dict[str, Any] = {}
+ for index in range(3):
+ agent = f"agent-{index}"
+ agent_dir = run_dir / agent
+ model = f"model-{index}"
+ source = "CLI"
+ summary = {
+ "agent": agent,
+ "model": model,
+ "model_source": source,
+ "scores": {},
+ "custom_scores": custom_scores,
+ "overall_score": 0.75,
+ "metric_set": "custom-only",
+ "metrics": [],
+ "dimensions": {},
+ "num_trials": 480,
+ "num_reward_rows": 480,
+ "pass_at_k": pass_summary,
+ **condition,
+ "job_failure": "aggregate failure: " + ("j" * 4_000),
+ "trial_failures": failures,
+ "trial_failure_details_total": 32,
+ "trial_failure_details_shown": 32,
+ "trial_failure_details_truncated": False,
+ }
+ _write_json(agent_dir / "with-skill" / "summary.json", summary)
+ _write_json(agent_dir / "without-skill" / "summary.json", summary)
+ _write_json(agent_dir / "lift.json", {})
+ _write_json(agent_dir / "custom_lift.json", custom_lift)
+ _write_json(agent_dir / "pass_at_k_lift.json", pass_lift)
+ _write_json(agent_dir / "security_attribution.json", security_attribution)
+ agents[agent] = {
+ "model": model,
+ "model_source": source,
+ "model_resolution": {"model": model, "source": source},
+ "with_skill": {},
+ "without_skill": {},
+ "custom_with_skill": custom_scores,
+ "custom_without_skill": custom_scores,
+ "dimensions_with_skill": {},
+ "dimensions_without_skill": {},
+ "lift": {},
+ "custom_lift": custom_lift,
+ "pass_at_k": {
+ "with_skill": pass_summary,
+ "without_skill": pass_summary,
+ "lift": pass_lift,
+ },
+ "security_attribution": security_attribution,
+ "agent_runtime_failures": {"with_skill": failures, "without_skill": failures},
+ "trial_failures": {"with_skill": failures, "without_skill": failures},
+ "failure_detail_metadata": {
+ key: {"details_total": 32, "details_shown": 32, "details_truncated": False}
+ for key in (
+ "with_skill_runtime",
+ "without_skill_runtime",
+ "with_skill_trials",
+ "without_skill_trials",
+ )
+ },
+ "job_failures": {"with_skill": summary["job_failure"], "without_skill": summary["job_failure"]},
+ "conditions": {"with_skill": condition, "without_skill": condition},
+ "execution_status": "failed",
+ "execution_errors": condition_errors,
+ "execution_error_details_total": 32,
+ "execution_error_details_shown": 32,
+ "execution_error_details_truncated": False,
+ "expected_attempts": 1_024,
+ "scored_attempts": 960,
+ "num_trials_with": 480,
+ "num_trials_without": 480,
+ "output_dir": str(agent_dir.resolve()),
+ }
+ run_config_agents[agent] = {"agent": agent, "model": model, "source": source}
+
+ comparison = {
+ "metrics": {
+ name: {agent: {"with_skill": 0.75, "without_skill": 0.25, "delta": 0.5} for agent in agents}
+ for name in metric_names
+ }
+ }
+ _write_json(run_dir / "comparison.json", comparison)
+ run_config = {
+ "config_file": "none",
+ "harbor": {
+ "environment": {"value": "docker", "source": "CLI"},
+ "n_attempts": 2,
+ "stop_on_pass": False,
+ "n_concurrent": 3,
+ "timeout_multiplier": 1.0,
+ "base_image_mode": "disabled",
+ "jobs_retained": True,
+ },
+ "provider": {"name": "openai", "model": "test-model"},
+ "task_source": "evals_json",
+ "grading": {"mode": "custom_only"},
+ "agents": run_config_agents,
+ }
+ return {
+ "skill_name": "demo",
+ "run_id": run_dir.name,
+ "run_dir": str(run_dir),
+ "result_path": str(run_dir / "result.json"),
+ "run_config": run_config,
+ "agents": agents,
+ "comparison": comparison,
+ "metric_set": "custom-only",
+ "metrics": [],
+ "attempt_policy": {
+ "max_attempts": 2,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "score_definition": "mean custom metrics",
+ },
+ "execution_status": "failed",
+ "execution_errors": condition_errors,
+ "execution_error_details_total": 32,
+ "execution_error_details_shown": 32,
+ "execution_error_details_truncated": False,
+ "error": condition_errors[:1],
+ "evaluator_version": __version__,
+ "dataset_snapshot": {
+ "schema_version": "1.0",
+ "evaluator_version": __version__,
+ "dataset_summary": {"total_tasks": 256},
+ "dataset_digest": "sha256:" + ("d" * 64),
+ "dataset_digest_algorithm": "skill-evaluator-dataset-snapshot/1",
+ },
+ "dataset_snapshot_path": str(run_dir / "dataset_snapshot.json"),
+ "dataset_summary": {"total_tasks": 256},
+ "dataset_digest": "sha256:" + ("d" * 64),
+ "dataset_digest_algorithm": "skill-evaluator-dataset-snapshot/1",
+ "report_status": "complete",
+ "duration_seconds": 1.0,
+ }
+
+
+def test_final_result_projects_three_max_detail_agents_to_bounded_artifact_references(tmp_path: Path) -> None:
+ from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory
+
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ run_dir = tmp_path / "20260825_120000_123_aaaaaaaaaaaa"
+ run_dir.mkdir()
+ (run_dir / "_harbor-jobs").mkdir()
+ (run_dir / "_harbor-tasks").mkdir()
+ result = _max_detail_result(run_dir)
+ _write_json(run_dir / "run_config.json", result["run_config"])
+ _write_json(run_dir / "result.json", {})
+ assert len(json.dumps(result, indent=2).encode("utf-8")) > 2 * 1024 * 1024
+
+ harbor_runner._finalize_harbor_artifacts(
+ run_dir_value=run_dir,
+ keep_requested=True,
+ result=result,
+ )
+
+ result_path = run_dir / "result.json"
+ encoded = result_path.read_bytes()
+ persisted = json.loads(encoded)
+ assert len(encoded) <= 2 * 1024 * 1024
+ assert result["agents"]["agent-0"]["pass_at_k"]["with_skill"]["cases"]
+ assert persisted["agents"]["agent-0"]["pass_at_k"]["with_skill"]["cases"] == {}
+ assert len(persisted["agents"]["agent-0"]["custom_with_skill"]) == 128
+ assert persisted["agents"]["agent-0"]["expected_attempts"] == 1_024
+ assert persisted["agents"]["agent-0"]["conditions"]["with_skill"]["execution_error_details_total"] == 32
+ assert persisted["agents"]["agent-0"]["security_attribution"]["case_details_total"] == 256
+ assert persisted["execution_errors"]
+ assert persisted["error"] == persisted["execution_errors"][:1]
+ assert persisted["dataset_digest"] == result["dataset_digest"]
+ assert persisted["result_path"] == result["result_path"]
+ projection = persisted["result_projection"]
+ assert projection == result["result_projection"]
+ assert projection["schema_version"] == "1.0"
+ assert projection["mode"] == "artifact_referenced"
+ reference = projection["agents"]["agent-0"]["with_skill_summary"]
+ referenced = run_dir / reference["path"]
+ assert reference["bytes"] == referenced.stat().st_size
+ assert reference["sha256"] == "sha256:" + hashlib.sha256(referenced.read_bytes()).hexdigest()
+ diagnostics: list[dict[str, Any]] = []
+ assert report_data._load_bounded_json(result_path, diagnostics, artifact="result") == persisted
+ assert diagnostics == []
+ regenerated = agent_eval_result_from_directory(
+ skill,
+ run_dir,
+ engine_result=None,
+ use_llm_judge=False,
+ )
+ assert regenerated is not None
+ assert regenerated.metadata["agent_eval"]["execution_status"] == "failed"
+
+
+def test_inline_final_result_round_trips_normal_agent_artifacts_for_disk_report(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.evaluation.tier3_report import agent_eval_result_from_directory
+
+ skill = tmp_path / "demo"
+ skill.mkdir()
+ run_dir = tmp_path / "20260825_120000_123_bbbbbbbbbbbb"
+ run_dir.mkdir()
+ agent_dir = run_dir / "opencode"
+ summary = {
+ "agent": "opencode",
+ "model": "test-model",
+ "model_source": "CLI",
+ "scores": {"overall": 1.0},
+ "custom_scores": {"quality": 0.75},
+ "overall_score": 1.0,
+ "metric_set": "skill-evaluator-default-v2",
+ "metrics": ["overall"],
+ "dimensions": {},
+ "num_trials": 1,
+ "num_reward_rows": 1,
+ "pass_at_k": {
+ "k": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "passed_cases": 1,
+ "failed_cases": 0,
+ "total_cases": 1,
+ "rate": 1.0,
+ "attempts_used": 1,
+ "max_attempts_possible": 1,
+ "cases": {"case-1": {"passed": True, "attempts": []}},
+ },
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "execution_error_details_total": 0,
+ "execution_error_details_shown": 0,
+ "execution_error_details_truncated": False,
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "job_failure": "",
+ "trial_failures": [],
+ }
+ _write_json(agent_dir / "with-skill" / "summary.json", summary)
+ baseline = {**summary, "execution_status": "skipped", "expected_attempts": 0, "scored_attempts": 0}
+ _write_json(agent_dir / "without-skill" / "summary.json", baseline)
+ reward_dir = agent_dir / "with-skill" / "trials" / "case-1_attempt001"
+ _write_json(
+ reward_dir / "reward.json",
+ {
+ "entry_id": "case-1",
+ "overall": 1.0,
+ "metric_set": "skill-evaluator-default-v2",
+ "metrics": ["overall"],
+ },
+ )
+ run_config = {
+ "config_file": "none",
+ "harbor": {
+ "environment": {"value": "docker", "source": "CLI"},
+ "n_attempts": 1,
+ "stop_on_pass": False,
+ "n_concurrent": 1,
+ "timeout_multiplier": 1.0,
+ "base_image_mode": "disabled",
+ "jobs_retained": True,
+ },
+ "provider": {"name": "openai", "model": "test-model"},
+ "task_source": "evals_json",
+ "grading": {"mode": "default_plus_custom"},
+ "agents": {"opencode": {"agent": "opencode", "model": "test-model", "source": "CLI"}},
+ }
+ result = {
+ "skill_name": skill.name,
+ "run_id": run_dir.name,
+ "run_dir": str(run_dir),
+ "result_path": str(run_dir / "result.json"),
+ "run_config": run_config,
+ "agents": {
+ "opencode": {
+ "model": "test-model",
+ "model_source": "CLI",
+ "model_resolution": {"model": "test-model", "source": "CLI"},
+ "with_skill": summary["scores"],
+ "without_skill": {},
+ "custom_with_skill": summary["custom_scores"],
+ "custom_without_skill": {},
+ "dimensions_with_skill": {},
+ "dimensions_without_skill": {},
+ "lift": {},
+ "custom_lift": {},
+ "pass_at_k": {"with_skill": summary["pass_at_k"], "without_skill": {}, "lift": {}},
+ "security_attribution": {},
+ "agent_runtime_failures": {"with_skill": [], "without_skill": []},
+ "trial_failures": {"with_skill": [], "without_skill": []},
+ "failure_detail_metadata": {},
+ "job_failures": {"with_skill": "", "without_skill": ""},
+ "conditions": {
+ "with_skill": {
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "execution_error_details_total": 0,
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ },
+ "without_skill": {
+ "execution_status": "skipped",
+ "execution_errors": [],
+ "execution_error_details_total": 0,
+ "expected_attempts": 0,
+ "scored_attempts": 0,
+ },
+ },
+ "output_dir": str(agent_dir.resolve()),
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "execution_error_details_total": 0,
+ "execution_error_details_shown": 0,
+ "execution_error_details_truncated": False,
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "num_trials_with": 1,
+ "num_trials_without": 0,
+ }
+ },
+ "metric_set": "skill-evaluator-default-v2",
+ "metrics": ["overall"],
+ "attempt_policy": {
+ "max_attempts": 1,
+ "pass_threshold": 0.5,
+ "stop_on_pass": False,
+ "score_definition": "overall",
+ },
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "error": [],
+ "report_status": "complete",
+ "duration_seconds": 1.0,
+ }
+ _write_json(run_dir / "run_config.json", run_config)
+ _write_json(run_dir / "result.json", {})
+
+ harbor_runner._finalize_harbor_artifacts(
+ run_dir_value=run_dir,
+ keep_requested=True,
+ result=result,
+ )
+
+ persisted = json.loads((run_dir / "result.json").read_text(encoding="utf-8"))
+ loaded_agents = report_data.load_agent_data(run_dir)
+ report_result = agent_eval_result_from_directory(
+ skill,
+ run_dir,
+ engine_result=None,
+ use_llm_judge=False,
+ )
+ assert persisted == result
+ assert persisted["result_projection"]["mode"] == "inline"
+ assert loaded_agents["opencode"]["pass_with_skill"]["cases"] == summary["pass_at_k"]["cases"]
+ assert report_result is not None
+ assert report_result.metadata["agent_eval"]["execution_status"] == "succeeded"
+
+
+def test_final_result_fails_closed_before_publishing_unreferenced_oversize_detail(tmp_path: Path) -> None:
+ result_path = tmp_path / "result.json"
+ result = {"execution_status": "failed", "execution_errors": [], "unreferenced": "x" * 2_100_000}
+
+ with pytest.raises(ValueError, match="Final Tier 3 result exceeds"):
+ harbor_runner._write_final_result(result_path, result)
+
+ assert not result_path.exists()
+ assert not list(tmp_path.glob(".result.json.*"))
+
+
+def test_final_result_projects_real_multi_agent_aggregate_errors_without_losing_exact_total(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ run_dir.mkdir()
+ agents: dict[str, Any] = {}
+ for agent_index in range(3):
+ agent = f"agent-{agent_index}"
+ conditions: dict[str, Any] = {}
+ for condition, directory in (("with_skill", "with-skill"), ("without_skill", "without-skill")):
+ failures = [
+ {
+ "trial": f"trial-{agent_index}-{condition}-{failure_index}",
+ "reason": f"reason-{agent_index}-{condition}-{failure_index}:" + ("🚀" * 1_800),
+ }
+ for failure_index in range(32)
+ ]
+ summary = harbor_collector._condition_execution_summary(
+ [],
+ expected_case_ids=[],
+ expected_cases=0,
+ n_attempts=1,
+ job_failure="",
+ runtime_failures=failures,
+ )
+ conditions[condition] = summary
+ _write_json(
+ run_dir / agent / directory / "summary.json",
+ {
+ "agent": agent,
+ "scores": {},
+ "custom_scores": {},
+ "pass_at_k": {},
+ "trial_failures": [],
+ "job_failure": "",
+ **summary,
+ },
+ )
+ agents[agent] = {
+ "pass_at_k": {"with_skill": {}, "without_skill": {}, "lift": {}},
+ "conditions": conditions,
+ "agent_runtime_failures": {"with_skill": [], "without_skill": []},
+ "trial_failures": {"with_skill": [], "without_skill": []},
+ "job_failures": {"with_skill": "", "without_skill": ""},
+ **harbor_collector._aggregate_execution(list(conditions.values())),
+ }
+
+ result: dict[str, Any] = {
+ "agents": agents,
+ **harbor_collector._aggregate_execution(list(agents.values())),
+ }
+ result["error"] = list(result["execution_errors"])
+ assert len(result["execution_errors"]) == 192
+ assert len(json.dumps(result, indent=2).encode("utf-8")) > 2 * 1024 * 1024
+
+ result_path = run_dir / "result.json"
+ harbor_runner._write_final_result(result_path, result)
+
+ persisted = json.loads(result_path.read_text(encoding="utf-8"))
+ assert result["execution_error_details_total"] == 192
+ assert len(result["execution_errors"]) == 192
+ assert persisted["execution_error_details_total"] == 192
+ assert persisted["execution_error_details_shown"] == 1
+ assert persisted["execution_error_details_truncated"] is True
+ assert persisted["error"] == persisted["execution_errors"][:1]
+ assert persisted["result_projection"]["omitted_root_detail_fields"] == ["error", "execution_errors"]
+ assert result_path.stat().st_size <= harbor_runner.FINAL_RESULT_MAX_BYTES
+ assert set(report_data.load_agent_data(run_dir)) == set(agents)
+
+
+def test_final_result_projection_preserves_aggregate_hidden_child_error_count(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ run_dir.mkdir()
+ _write_json(run_dir / "opencode" / "with-skill" / "summary.json", {})
+ _write_json(run_dir / "opencode" / "without-skill" / "summary.json", {})
+ child_summaries = [
+ {
+ "execution_status": "failed",
+ "execution_errors": ["shared visible error"],
+ "execution_error_details_total": hidden_total,
+ "execution_error_details_shown": 1,
+ "execution_error_details_truncated": True,
+ "expected_attempts": 1,
+ "scored_attempts": 0,
+ }
+ for hidden_total in (300, 2)
+ ]
+ agent = harbor_collector._aggregate_execution(child_summaries)
+ agent["conditions"] = {
+ "with_skill": child_summaries[0],
+ "without_skill": child_summaries[1],
+ }
+ result: dict[str, Any] = {
+ "agents": {"opencode": agent},
+ **harbor_collector._aggregate_execution([agent]),
+ }
+ result["error"] = list(result["execution_errors"])
+
+ result_path = run_dir / "result.json"
+ harbor_runner._write_final_result(result_path, result)
+
+ persisted = json.loads(result_path.read_text(encoding="utf-8"))
+ assert result["execution_error_details_total"] == 302
+ assert result["execution_error_details_truncated"] is True
+ assert persisted["execution_error_details_total"] == 302
+ assert persisted["execution_error_details_shown"] == 1
+ assert persisted["execution_error_details_truncated"] is True
+ assert persisted["agents"]["opencode"]["execution_error_details_total"] == 302
+ assert persisted["agents"]["opencode"]["execution_error_details_truncated"] is True
+
+
+def test_result_artifact_reference_rejects_intermediate_symlink(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ _write_json(run_dir / "inside" / "summary.json", {"scores": {}})
+ (run_dir / "agent").mkdir()
+ try:
+ (run_dir / "agent" / "with-skill").symlink_to("../inside", target_is_directory=True)
+ except OSError as exc:
+ pytest.skip(f"symlinks unavailable: {exc}")
+
+ assert (
+ harbor_runner._result_artifact_reference(
+ run_dir,
+ Path("agent/with-skill/summary.json"),
+ )
+ is None
+ )
+
+
+def test_result_artifact_reference_rejects_final_symlink_and_hardlink(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ source = run_dir / "source.json"
+ _write_json(source, {"scores": {}})
+ target_dir = run_dir / "agent" / "with-skill"
+ target_dir.mkdir(parents=True)
+ linked = target_dir / "summary.json"
+ try:
+ linked.symlink_to(source)
+ except OSError as exc:
+ pytest.skip(f"symlinks unavailable: {exc}")
+ assert harbor_runner._result_artifact_reference(run_dir, Path("agent/with-skill/summary.json")) is None
+
+ linked.unlink()
+ try:
+ os.link(source, linked)
+ except OSError as exc:
+ pytest.skip(f"hardlinks unavailable: {exc}")
+ assert harbor_runner._result_artifact_reference(run_dir, Path("agent/with-skill/summary.json")) is None
+
+
+def test_result_artifact_reference_rejects_oversize_or_invalid_json(tmp_path: Path) -> None:
+ run_dir = tmp_path / "run"
+ artifact = run_dir / "agent" / "with-skill" / "summary.json"
+ artifact.parent.mkdir(parents=True)
+ artifact.write_text(json.dumps({"value": "x" * harbor_runner.FINAL_RESULT_MAX_BYTES}), encoding="utf-8")
+ relative = Path("agent/with-skill/summary.json")
+ assert harbor_runner._result_artifact_reference(run_dir, relative) is None
+
+ artifact.write_text("{not-json", encoding="utf-8")
+ assert harbor_runner._result_artifact_reference(run_dir, relative) is None
diff --git a/tests/test_harbor_runner_environment.py b/tests/test_harbor_runner_environment.py
index 5ef717b6..be4acb13 100644
--- a/tests/test_harbor_runner_environment.py
+++ b/tests/test_harbor_runner_environment.py
@@ -20,6 +20,7 @@
from skillevaluator.provider_config import ProviderConfig, ProviderConfigurationError, resolve_llm_provider
from skillevaluator.tier3.harbor import runner, runtime_preflight
from skillevaluator.tier3.harbor.adapter import _verifier_env_vars
+from skillevaluator.tier3_environments import HARBOR_NATIVE_ENV_MODES
def _provider(provider: str = "openai") -> ProviderConfig:
@@ -189,6 +190,30 @@ def test_skill_config_cannot_alias_operator_owned_credentials(
assert source_name in errors[0]
+@pytest.mark.parametrize(
+ "runtime_env",
+ [
+ {"SAFE_ALIAS": "${PROJECT_DEPLOY_TOKEN}"},
+ {"PROJECT_DEPLOY_TOKEN": "${PROJECT_DEPLOY_TOKEN}"},
+ {"safe_alias": "${ProjectDeployToken}"},
+ ],
+)
+def test_skill_config_cannot_stage_conventional_unlisted_secret_names(
+ monkeypatch: pytest.MonkeyPatch,
+ runtime_env: dict[str, str],
+) -> None:
+ for value in runtime_env.values():
+ source_name = value.removeprefix("${").removesuffix("}")
+ monkeypatch.setenv(source_name, "synthetic-secret")
+
+ resolved, errors = runner._resolve_runtime_env(runtime_env)
+
+ assert resolved == {}
+ assert errors
+ assert "operator-owned" in errors[0] or "host process" in errors[0]
+ assert "synthetic-secret" not in " ".join(errors)
+
+
@pytest.mark.parametrize("source_name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
def test_skill_config_cannot_alias_judge_model_with_default_expansion(
monkeypatch: pytest.MonkeyPatch,
@@ -220,7 +245,10 @@ def test_skill_config_cannot_control_evaluator_or_judge_routing(name: str) -> No
assert errors and "host process" in errors[0]
-@pytest.mark.parametrize("name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
+@pytest.mark.parametrize(
+ "name",
+ ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL", "LLM_JUDGE_FALLBACK_MODELS"],
+)
@pytest.mark.parametrize("provider_name", ["openai", "openai-compatible", "anthropic", "bedrock", "nv_build"])
@pytest.mark.parametrize(
("configured", "expected"),
@@ -266,15 +294,15 @@ def test_runtime_env_preserves_platform_expansion_for_non_owned_percent_referenc
def expandvars(value: str) -> str:
expanded.append(value)
- return value.replace("%HOST_AGENT_TOKEN%", "host-value")
+ return value.replace("%HOST_AGENT_LABEL%", "host-value")
monkeypatch.setattr(runner.os.path, "expandvars", expandvars)
- resolved, errors = runner._resolve_runtime_env({"SAFE_ALIAS": "%HOST_AGENT_TOKEN%"})
+ resolved, errors = runner._resolve_runtime_env({"SAFE_ALIAS": "%HOST_AGENT_LABEL%"})
assert errors == []
assert resolved == {"SAFE_ALIAS": "host-value"}
- assert expanded == ["%HOST_AGENT_TOKEN%"]
+ assert expanded == ["%HOST_AGENT_LABEL%"]
def test_custom_only_does_not_force_the_standard_judge_model_into_custom_verifiers() -> None:
@@ -316,6 +344,22 @@ def test_standard_judge_override_occupies_the_highest_precedence_verifier_key(
assert runner._job_judge_verifier_env(provider_env, "default") == expected
+@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
+def test_standard_judge_fallback_is_forwarded_only_through_the_verifier_job(
+ grading_mode: str,
+) -> None:
+ provider_env = {"LLM_JUDGE_FALLBACK_MODELS": "fallback-one,fallback-two"}
+
+ assert runner._job_judge_verifier_env(provider_env, grading_mode) == {
+ "LLM_JUDGE_FALLBACK_MODELS": "${LLM_JUDGE_FALLBACK_MODELS}",
+ }
+ assert runner._job_judge_subprocess_env(provider_env, grading_mode) == {
+ "LLM_JUDGE_FALLBACK_MODELS": "fallback-one,fallback-two",
+ }
+ assert runner._job_judge_verifier_env(provider_env, "custom_only") == {}
+ assert runner._job_judge_subprocess_env(provider_env, "custom_only") == {}
+
+
@pytest.mark.parametrize(
"host_env",
[
@@ -969,9 +1013,12 @@ def test_run_harbor_eval_stages_per_agent_credential_trees(
provider = _provider("nv_build")
emitted: list[tuple[str, bool, dict[str, str], dict[str, str]]] = []
launched: dict[str, tuple[str, str, dict[str, str], dict[str, str]]] = {}
+ launched_task_selectors: dict[str, list[str]] = {}
+ collected: dict[str, object] = {}
monkeypatch.setenv("LLM_JUDGE_MODEL", "legacy-judge-model")
monkeypatch.setenv("SKILL_EVAL_JUDGE_MODEL", "judge-model")
+ monkeypatch.setenv("LLM_JUDGE_FALLBACK_MODELS", "fallback-one,fallback-two")
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: provider)
monkeypatch.setattr(
@@ -985,6 +1032,11 @@ def test_run_harbor_eval_stages_per_agent_credential_trees(
def emit(_skill, target, *, with_skill, runtime_env, **_kwargs):
task = target / "case-001"
task.mkdir(parents=True)
+ (task / "task.toml").write_text(
+ 'schema_version = "1.3"\n\n[task]\nname = "publisher/presentation-name"\n\n'
+ '[metadata]\nentry_id = "logical-case-001"\n',
+ encoding="utf-8",
+ )
emitted.append(
(
str(target.relative_to(tmp_path / "results")),
@@ -996,6 +1048,8 @@ def emit(_skill, target, *, with_skill, runtime_env, **_kwargs):
return [task]
def launch(**kwargs):
+ assert (kwargs["jobs_dir"].parent / "dataset_snapshot.json").is_file()
+ launched_task_selectors[kwargs["agent"]] = list(kwargs["task_names"])
launched[kwargs["agent"]] = (
str(kwargs["with_skill"].relative_to(tmp_path / "results")),
str(kwargs["baseline"].relative_to(tmp_path / "results")),
@@ -1019,7 +1073,10 @@ def launch(**kwargs):
monkeypatch.setattr(
runner,
"collect_harbor_results",
- lambda **_kwargs: {"execution_status": "complete", "execution_errors": [], "metrics": [], "agents": {}},
+ lambda **kwargs: (
+ collected.update(kwargs)
+ or {"execution_status": "complete", "execution_errors": [], "metrics": [], "agents": {}}
+ ),
)
monkeypatch.setattr(runner, "render_agent_eval_html_report", lambda *_args, **_kwargs: tmp_path / "report.html")
result = runner.run_harbor_eval(
@@ -1054,16 +1111,23 @@ def launch(**kwargs):
for _path, _with_skill, runtime_env, verifier_env in emitted:
assert "LLM_JUDGE_MODEL" not in runtime_env
assert "SKILL_EVAL_JUDGE_MODEL" not in runtime_env
+ assert "LLM_JUDGE_FALLBACK_MODELS" not in runtime_env
assert "LLM_JUDGE_MODEL" not in verifier_env
assert "SKILL_EVAL_JUDGE_MODEL" not in verifier_env
+ assert verifier_env["LLM_JUDGE_FALLBACK_MODELS"] == "${LLM_JUDGE_FALLBACK_MODELS}"
assert launched["opencode"][0].endswith("_harbor-tasks/opencode/with")
assert launched["claude-code"][1].endswith("_harbor-tasks/claude-code/without")
+ assert launched_task_selectors == {"opencode": ["case-001"], "claude-code": ["case-001"]}
+ assert collected["expected_case_ids"] == ["logical-case-001"]
+ assert collected["case_id_by_task_selector"] == {"case-001": "logical-case-001"}
assert launched["opencode"][2]["LLM_JUDGE_MODEL"] == "legacy-judge-model"
assert launched["opencode"][2]["SKILL_EVAL_JUDGE_MODEL"] == "legacy-judge-model"
+ assert launched["opencode"][2]["LLM_JUDGE_FALLBACK_MODELS"] == "fallback-one,fallback-two"
assert "ANTHROPIC_API_KEY" not in launched["opencode"][2]
assert "OPENAI_API_KEY" not in launched["opencode"][2]
assert launched["claude-code"][2]["NVIDIA_API_KEY"] == "provider-key"
assert launched["opencode"][3] == {
+ "LLM_JUDGE_FALLBACK_MODELS": "${LLM_JUDGE_FALLBACK_MODELS}",
"LLM_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
"SKILL_EVAL_JUDGE_MODEL": "${LLM_JUDGE_MODEL}",
}
@@ -1079,9 +1143,12 @@ def launch(**kwargs):
@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
-@pytest.mark.parametrize("judge_alias", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
+@pytest.mark.parametrize(
+ "judge_alias",
+ ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL", "LLM_JUDGE_FALLBACK_MODELS"],
+)
@pytest.mark.parametrize("judge_scope", ["task", "step"])
-def test_native_runtime_judge_selection_skips_fallback_probe_through_real_staging(
+def test_native_runtime_rejects_task_owned_judge_selection_through_real_staging(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
grading_mode: str,
@@ -1173,29 +1240,10 @@ def render_report(_skill_path: Path, output_dir: Path, **_kwargs) -> Path:
agent_runtime_preflight=False,
)
- assert "error" not in result
- assert probed_models == ["nvidia/nemotron-3-nano-30b-a3b"]
- assert result["execution_status"] == "complete"
- assert result["report_status"] == "complete"
+ assert result["execution_status"] == "failed"
+ assert any("standard grading" in error and "environment control" in error for error in result["error"])
assert result["run_config"]["task_source"] == "native_harbor"
- assert result["run_config"]["judge"]["catalog_verification"] == "inconclusive"
- assert result["run_config"]["judge"]["effective_model_source"] == "native_harbor_runtime"
- assert result["run_config"]["credential_validation"]["status"] == "degraded"
- standard_target = next(
- target
- for target in result["run_config"]["credential_validation"]["targets"]
- if target["labels"] == ["standard grader"]
- )
- assert standard_target["status"] == "inconclusive"
- assert standard_target["model"] is None
-
- assert len(staged_tasks) == 1
- staged_config = tomllib.loads((staged_tasks[0] / "task.toml").read_text(encoding="utf-8"))
- if judge_scope == "task":
- staged_judge_env = staged_config["verifier"]["env"]
- else:
- staged_judge_env = staged_config["steps"][0]["verifier"]["env"]
- assert staged_judge_env[judge_alias] == "authored-native-judge"
+ assert staged_tasks == []
def test_custom_only_native_verifier_placeholders_use_task_fallbacks_not_standard_override(
@@ -1228,6 +1276,7 @@ def test_custom_only_native_verifier_placeholders_use_task_fallbacks_not_standar
monkeypatch.setenv("LLM_JUDGE_MODEL", "standard-legacy-model")
monkeypatch.setenv("SKILL_EVAL_JUDGE_MODEL", "standard-canonical-model")
+ monkeypatch.setenv("LLM_JUDGE_FALLBACK_MODELS", "standard-fallback-one,standard-fallback-two")
monkeypatch.setattr(runner, "resolve_llm_provider", lambda: _provider("nv_build"))
monkeypatch.setattr(
runner,
@@ -1282,6 +1331,8 @@ def launch(**kwargs):
)
authored_env = task_config["verifier"]["env"]
run_env = launched["run_env"]
+ assert "LLM_JUDGE_FALLBACK_MODELS" not in authored_env
+ assert "LLM_JUDGE_FALLBACK_MODELS" not in run_env
task_fallbacks = {
"LLM_JUDGE_MODEL": "task-legacy-model",
"SKILL_EVAL_JUDGE_MODEL": "task-canonical-model",
@@ -1343,6 +1394,7 @@ def test_harbor_subprocess_environment_excludes_arbitrary_host_secrets(
host_environment = {
"PATH": os.environ["PATH"],
"HOME": str(tmp_path),
+ "USERPROFILE": str(tmp_path),
"TMPDIR": str(tmp_path / "tmp"),
"LANG": "C.UTF-8",
"DOCKER_HOST": "unix:///safe/docker.sock",
@@ -1368,7 +1420,9 @@ def test_harbor_subprocess_environment_excludes_arbitrary_host_secrets(
"TMPDIR": str(tmp_path / "tmp"),
"LANG": "C.UTF-8",
"DOCKER_HOST": "unix:///safe/docker.sock",
+ "SSH_AUTH_SOCK": "/private/agent.sock",
"SERVICE_TOKEN": "declared-runtime-secret",
+ "USERPROFILE": str(tmp_path),
**provider_env,
}
@@ -1401,6 +1455,541 @@ def test_harbor_subprocess_environment_includes_only_selected_backend_credential
assert "MODAL_TOKEN_SECRET" not in environment
+def test_harbor_subprocess_environment_preserves_trusted_proxy_and_tls_controls(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ network_environment = {
+ "HTTP_PROXY": "http://proxy.example:8080",
+ "HTTPS_PROXY": "https://proxy.example:8443",
+ "NO_PROXY": "localhost,127.0.0.1",
+ "ALL_PROXY": "socks5://proxy.example:1080",
+ "http_proxy": "http://legacy-proxy.example:8080",
+ "https_proxy": "https://legacy-proxy.example:8443",
+ "no_proxy": "metadata.internal",
+ "all_proxy": "socks5://legacy-proxy.example:1080",
+ "SSL_CERT_FILE": "/etc/ssl/custom.pem",
+ "SSL_CERT_DIR": "/etc/ssl/custom",
+ "REQUESTS_CA_BUNDLE": "/etc/ssl/requests.pem",
+ "CURL_CA_BUNDLE": "/etc/ssl/curl.pem",
+ }
+ monkeypatch.setattr(
+ runner.os,
+ "environ",
+ {"PATH": "/usr/bin", "HOME": "/home/test", **network_environment, "UNRELATED_SECRET": "no"},
+ )
+ provider = _provider()
+
+ environment = runner._harbor_subprocess_environment(
+ env_mode="e2b",
+ provider=provider,
+ configured_runtime_env={},
+ provider_env=runner._provider_environment(provider),
+ )
+
+ assert {name: environment[name] for name in network_environment} == network_environment
+ assert "UNRELATED_SECRET" not in environment
+
+
+def test_harbor_backend_environment_allowlist_covers_every_native_022_mode() -> None:
+ expected = {
+ "docker": {
+ "COMPOSE_ANSI",
+ "COMPOSE_HTTP_TIMEOUT",
+ "COMPOSE_IGNORE_ORPHANS",
+ "COMPOSE_PARALLEL_LIMIT",
+ "COMPOSE_PROGRESS",
+ "COMPOSE_STATUS_STDOUT",
+ "DOCKER_API_VERSION",
+ "DOCKER_AUTH_CONFIG",
+ "DOCKER_CERT_PATH",
+ "DOCKER_CONFIG",
+ "DOCKER_CONTEXT",
+ "DOCKER_CUSTOM_HEADERS",
+ "DOCKER_DEFAULT_PLATFORM",
+ "DOCKER_HOST",
+ "DOCKER_TLS",
+ "DOCKER_TLS_VERIFY",
+ "SSH_AUTH_SOCK",
+ },
+ "daytona": {
+ "DAYTONA_API_KEY",
+ "DAYTONA_API_URL",
+ "DAYTONA_HAPPY_EYEBALLS_DELAY",
+ "DAYTONA_JWT_TOKEN",
+ "DAYTONA_ORGANIZATION_ID",
+ "DAYTONA_SERVER_URL",
+ "DAYTONA_TARGET",
+ },
+ "e2b": {"E2B_API_KEY", "E2B_API_URL", "E2B_DOMAIN", "E2B_SANDBOX_URL"},
+ "modal": {
+ "MODAL_CONFIG_PATH",
+ "MODAL_ENVIRONMENT",
+ "MODAL_OVERRIDE_HEADERS",
+ "MODAL_PROFILE",
+ "MODAL_SERVER_URL",
+ "MODAL_TOKEN_ID",
+ "MODAL_TOKEN_SECRET",
+ },
+ "runloop": {"RUNLOOP_API_KEY", "RUNLOOP_BASE_URL", "RUNLOOP_CUSTOM_HEADERS"},
+ "langsmith": {
+ "LANGCHAIN_API_KEY",
+ "LANGCHAIN_ENDPOINT",
+ "LANGSMITH_API_KEY",
+ "LANGSMITH_CONFIG_FILE",
+ "LANGSMITH_ENDPOINT",
+ "LANGSMITH_PROFILE",
+ "LANGSMITH_SANDBOX_API_URL",
+ "LANGSMITH_WORKSPACE_ID",
+ },
+ "ec2": {
+ "AWS_ACCOUNT_ID",
+ "AWS_ACCOUNT_ID_ENDPOINT_MODE",
+ "AWS_ACCESS_KEY_ID",
+ "AWS_AUTH_SCHEME_PREFERENCE",
+ "AWS_CA_BUNDLE",
+ "AWS_CONFIG_FILE",
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN",
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI",
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
+ "AWS_CREDENTIAL_EXPIRATION",
+ "AWS_CREDENTIAL_FILE",
+ "AWS_CSM_CLIENT_ID",
+ "AWS_CSM_ENABLED",
+ "AWS_CSM_HOST",
+ "AWS_CSM_PORT",
+ "AWS_DATA_PATH",
+ "AWS_DEFAULT_PROFILE",
+ "AWS_DEFAULT_REGION",
+ "AWS_DEFAULTS_MODE",
+ "AWS_DISABLE_HOST_PREFIX_INJECTION",
+ "AWS_DISABLE_REQUEST_COMPRESSION",
+ "AWS_EC2_METADATA_DISABLED",
+ "AWS_EC2_METADATA_SERVICE_ENDPOINT",
+ "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
+ "AWS_EC2_METADATA_V1_DISABLED",
+ "AWS_ENDPOINT_DISCOVERY_ENABLED",
+ "AWS_ENDPOINT_URL",
+ "AWS_ENDPOINT_URL_EC2",
+ "AWS_ENDPOINT_URL_SIGNIN",
+ "AWS_ENDPOINT_URL_SSO",
+ "AWS_ENDPOINT_URL_SSO_OIDC",
+ "AWS_ENDPOINT_URL_STS",
+ "AWS_EXECUTION_ENV",
+ "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS",
+ "AWS_IMDS_USE_IPV6",
+ "AWS_LOGIN_CACHE_DIRECTORY",
+ "AWS_MAX_ATTEMPTS",
+ "AWS_METADATA_SERVICE_NUM_ATTEMPTS",
+ "AWS_METADATA_SERVICE_TIMEOUT",
+ "AWS_NEW_RETRIES_2026",
+ "AWS_PROFILE",
+ "AWS_REGION",
+ "AWS_RETRY_MODE",
+ "AWS_REQUEST_CHECKSUM_CALCULATION",
+ "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES",
+ "AWS_RESPONSE_CHECKSUM_VALIDATION",
+ "AWS_ROLE_ARN",
+ "AWS_ROLE_SESSION_NAME",
+ "AWS_SDK_LOAD_CONFIG",
+ "AWS_SDK_UA_APP_ID",
+ "AWS_SECRET_ACCESS_KEY",
+ "AWS_SECURITY_TOKEN",
+ "AWS_SESSION_TOKEN",
+ "AWS_SHARED_CREDENTIALS_FILE",
+ "AWS_SIGV4A_SIGNING_REGION_SET",
+ "AWS_STS_REGIONAL_ENDPOINTS",
+ "AWS_USE_DUALSTACK_ENDPOINT",
+ "AWS_USE_FIPS_ENDPOINT",
+ "AWS_WEB_IDENTITY_TOKEN_FILE",
+ "BOTOCORE_TCP_KEEPALIVE",
+ "SSH_AUTH_SOCK",
+ },
+ "gke": {
+ "CLOUDSDK_ACTIVE_CONFIG_NAME",
+ "CLOUDSDK_AUTH_ACCESS_TOKEN",
+ "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE",
+ "CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT",
+ "CLOUDSDK_CONFIG",
+ "CLOUDSDK_CORE_ACCOUNT",
+ "CLOUDSDK_CORE_PROJECT",
+ "GCP_PROJECT",
+ "GOOGLE_APPLICATION_CREDENTIALS",
+ "GOOGLE_CLOUD_PROJECT",
+ "GOOGLE_CLOUD_QUOTA_PROJECT",
+ "KUBECONFIG",
+ },
+ "ack": {
+ "COMPOSE_ANSI",
+ "COMPOSE_HTTP_TIMEOUT",
+ "COMPOSE_IGNORE_ORPHANS",
+ "COMPOSE_PARALLEL_LIMIT",
+ "COMPOSE_PROGRESS",
+ "COMPOSE_STATUS_STDOUT",
+ "DOCKER_API_VERSION",
+ "DOCKER_AUTH_CONFIG",
+ "DOCKER_CERT_PATH",
+ "DOCKER_CONFIG",
+ "DOCKER_CONTEXT",
+ "DOCKER_CUSTOM_HEADERS",
+ "DOCKER_DEFAULT_PLATFORM",
+ "DOCKER_HOST",
+ "DOCKER_TLS",
+ "DOCKER_TLS_VERIFY",
+ "KUBECONFIG",
+ "KUBERNETES_SERVICE_HOST",
+ "KUBERNETES_SERVICE_PORT",
+ "SSH_AUTH_SOCK",
+ },
+ "openshift": {"KUBECONFIG"},
+ "novita": {
+ "NOVITA_ACCESS_TOKEN",
+ "NOVITA_API_KEY",
+ "NOVITA_API_URL",
+ "NOVITA_BASE_URL",
+ "NOVITA_DOMAIN",
+ "NOVITA_SANDBOX_URL",
+ },
+ "apple-container": set(),
+ "singularity": {
+ "APPTAINER_AUTHFILE",
+ "APPTAINER_CONFIGDIR",
+ "APPTAINER_DOCKER_PASSWORD",
+ "APPTAINER_DOCKER_USERNAME",
+ "SINGULARITY_AUTHFILE",
+ "SINGULARITY_CONFIGDIR",
+ "SINGULARITY_DOCKER_PASSWORD",
+ "SINGULARITY_DOCKER_USERNAME",
+ },
+ "islo": {"ISLO_API_KEY", "ISLO_API_URL", "ISLO_COMPUTE_URL"},
+ "tensorlake": {
+ "TENSORLAKE_API_KEY",
+ "TENSORLAKE_API_URL",
+ "TENSORLAKE_ORGANIZATION_ID",
+ "TENSORLAKE_PAT",
+ "TENSORLAKE_PROJECT_ID",
+ "TENSORLAKE_SANDBOX_PROXY_URL",
+ },
+ "cwsandbox": {"CWSANDBOX_API_KEY", "CWSANDBOX_BASE_URL"},
+ "wandb": {"NETRC", "WANDB_API_KEY", "WANDB_BASE_URL", "WANDB_ENTITY", "WANDB_PROJECT"},
+ "use-computer": {
+ "USE_COMPUTER_API_KEY",
+ "USE_COMPUTER_HOST",
+ "USE_COMPUTER_SNAPSHOT",
+ "USE_COMPUTER_VERSION",
+ },
+ "cua-cloud": {
+ "CUA_BASE_URL",
+ "CUA_CLIENT_ID",
+ "CUA_CLIENT_SECRET",
+ "CUA_CLOUD_NAMESPACE",
+ "CUA_CLOUD_STARTUP_COMMAND",
+ "CUA_TOKEN_URL",
+ },
+ "blaxel": {
+ "BL_API_KEY",
+ "BL_API_VERSION",
+ "BL_CLIENT_CREDENTIALS",
+ "BL_ENV",
+ "BL_REGION",
+ "BL_WORKSPACE",
+ },
+ "opensandbox": {"OPENSANDBOX_API_KEY", "OPENSANDBOX_DOMAIN"},
+ "beam": {
+ "API_HOST",
+ "API_PORT",
+ "BEAM_TOKEN",
+ "GATEWAY_HOST",
+ "GATEWAY_PORT",
+ "INTERNAL_API_HOST",
+ "INTERNAL_API_PORT",
+ "REALTIME_HOST",
+ },
+ "skypilot": {
+ "COMPOSE_ANSI",
+ "COMPOSE_HTTP_TIMEOUT",
+ "COMPOSE_IGNORE_ORPHANS",
+ "COMPOSE_PARALLEL_LIMIT",
+ "COMPOSE_PROGRESS",
+ "COMPOSE_STATUS_STDOUT",
+ "DOCKER_API_VERSION",
+ "DOCKER_AUTH_CONFIG",
+ "DOCKER_CERT_PATH",
+ "DOCKER_CONFIG",
+ "DOCKER_CONTEXT",
+ "DOCKER_CUSTOM_HEADERS",
+ "DOCKER_DEFAULT_PLATFORM",
+ "DOCKER_HOST",
+ "DOCKER_TLS",
+ "DOCKER_TLS_VERIFY",
+ "HARBOR_SKYPILOT_REGISTRY",
+ "SKYPILOT_API_SERVER_ENDPOINT",
+ "SKYPILOT_GLOBAL_CONFIG",
+ "SKYPILOT_PROJECT_CONFIG",
+ "SKYPILOT_SERVICE_ACCOUNT_TOKEN",
+ "SSH_AUTH_SOCK",
+ },
+ "hf-sandbox": {"HF_ENDPOINT", "HF_HOME", "HF_TOKEN", "HF_TOKEN_PATH", "HUGGING_FACE_HUB_TOKEN"},
+ "hyperbrowser": {
+ "COMPOSE_ANSI",
+ "COMPOSE_HTTP_TIMEOUT",
+ "COMPOSE_IGNORE_ORPHANS",
+ "COMPOSE_PARALLEL_LIMIT",
+ "COMPOSE_PROGRESS",
+ "COMPOSE_STATUS_STDOUT",
+ "DOCKER_API_VERSION",
+ "DOCKER_AUTH_CONFIG",
+ "DOCKER_CERT_PATH",
+ "DOCKER_CONFIG",
+ "DOCKER_CONTEXT",
+ "DOCKER_CUSTOM_HEADERS",
+ "DOCKER_DEFAULT_PLATFORM",
+ "DOCKER_HOST",
+ "DOCKER_TLS",
+ "DOCKER_TLS_VERIFY",
+ "HYPERBROWSER_API_KEY",
+ "HYPERBROWSER_BASE_URL",
+ "SSH_AUTH_SOCK",
+ },
+ "vercel": {"VERCEL_OIDC_TOKEN", "VERCEL_PROJECT_ID", "VERCEL_TEAM_ID", "VERCEL_TOKEN"},
+ }
+
+ assert set(expected) == HARBOR_NATIVE_ENV_MODES
+ assert {mode: set(names) for mode, names in runner._HARBOR_ENV_MODE_VARS.items()} == expected
+
+
+def test_direct_run_returns_configuration_error_for_cyclic_environment_kwargs(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ cyclic: dict[str, object] = {}
+ cyclic["nested"] = cyclic
+ monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
+ monkeypatch.setattr(runner, "load_evals_config", lambda _path: ({}, None))
+
+ result = runner._run_harbor_eval_impl(
+ tmp_path,
+ ["opencode"],
+ environment_kwargs={"options": cyclic},
+ _evaluator_skill_path=tmp_path,
+ )
+
+ assert result == {"error": ["Invalid --environment-kwarg: must not contain cyclic values"]}
+
+
+@pytest.mark.parametrize("timeout_multiplier", [float("nan"), float("inf"), float("-inf")])
+def test_direct_run_rejects_nonfinite_timeout_multiplier_before_creating_results(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ timeout_multiplier: float,
+) -> None:
+ monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
+ monkeypatch.setattr(
+ runner,
+ "load_evals_config",
+ lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
+ )
+
+ result = runner._run_harbor_eval_impl(
+ tmp_path,
+ ["opencode"],
+ timeout_multiplier=timeout_multiplier,
+ output_dir=tmp_path / "results",
+ _evaluator_skill_path=tmp_path,
+ )
+
+ assert result == {"error": ["timeout_multiplier must be a finite number greater than 0"]}
+ assert not (tmp_path / "results").exists()
+
+
+def test_direct_run_rejects_overflowing_timeout_multiplier_before_creating_results(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
+ monkeypatch.setattr(
+ runner,
+ "load_evals_config",
+ lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
+ )
+
+ result = runner._run_harbor_eval_impl(
+ tmp_path,
+ ["opencode"],
+ timeout_multiplier=10**1000,
+ output_dir=tmp_path / "results",
+ _evaluator_skill_path=tmp_path,
+ )
+
+ assert result == {"error": ["timeout_multiplier must be a finite number greater than 0"]}
+ assert not (tmp_path / "results").exists()
+
+
+def test_direct_run_rejects_finite_multiplier_that_overflows_default_timeouts(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
+ monkeypatch.setattr(
+ runner,
+ "load_evals_config",
+ lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
+ )
+
+ result = runner._run_harbor_eval_impl(
+ tmp_path,
+ ["opencode"],
+ timeout_multiplier=1e308,
+ output_dir=tmp_path / "results",
+ _evaluator_skill_path=tmp_path,
+ )
+
+ assert result == {"error": ["timeout_multiplier must yield finite Harbor timeouts"]}
+ assert not (tmp_path / "results").exists()
+
+
+def test_direct_run_rejects_overflowing_pass_threshold_before_creating_results(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
+ monkeypatch.setattr(
+ runner,
+ "load_evals_config",
+ lambda _path: ({"harbor": {"task_source": "evals_json"}}, None),
+ )
+
+ result = runner._run_harbor_eval_impl(
+ tmp_path,
+ ["opencode"],
+ pass_threshold=10**1000,
+ output_dir=tmp_path / "results",
+ _evaluator_skill_path=tmp_path,
+ )
+
+ assert result == {"error": ["pass_threshold must be a finite number between 0.0 and 1.0"]}
+ assert not (tmp_path / "results").exists()
+
+
+@pytest.mark.parametrize("setup_key", ["pre_agent_setup", "setup_commands"])
+def test_paired_run_rejects_normalized_skill_owned_setup_before_environment_preflight(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ setup_key: str,
+) -> None:
+ skill = tmp_path / "skill"
+ evals = skill / "evals"
+ evals.mkdir(parents=True)
+ (evals / "config.yml").write_text(
+ f"schema_version: 1\nharbor:\n {setup_key}: echo ready\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
+
+ def unexpected_preflight(**_kwargs: object) -> list[str]:
+ pytest.fail("paired skill-owned setup reached environment preflight")
+
+ monkeypatch.setattr(runner, "_check_prerequisites", unexpected_preflight)
+
+ result = runner._run_harbor_eval_impl(
+ skill,
+ ["opencode"],
+ output_dir=tmp_path / "results",
+ _evaluator_skill_path=skill,
+ )
+
+ assert result == {
+ "error": [
+ "harbor.pre_agent_setup/setup_commands cannot run in a paired evaluation; "
+ "use --skip-baseline for a with-skill-only run"
+ ]
+ }
+ assert not (tmp_path / "results").exists()
+
+
+@pytest.mark.parametrize("setup_key", ["pre_agent_setup", "setup_commands"])
+def test_skip_baseline_keeps_normalized_skill_owned_setup_enabled(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ setup_key: str,
+) -> None:
+ skill = tmp_path / "skill"
+ evals = skill / "evals"
+ evals.mkdir(parents=True)
+ (evals / "config.yml").write_text(
+ f"schema_version: 1\nharbor:\n {setup_key}: echo ready\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setattr(runner, "resolve_llm_provider", _provider)
+ monkeypatch.setattr(runner, "_check_prerequisites", lambda **_kwargs: ["preflight sentinel"])
+
+ result = runner._run_harbor_eval_impl(
+ skill,
+ ["opencode"],
+ skip_baseline=True,
+ output_dir=tmp_path / "results",
+ _evaluator_skill_path=skill,
+ )
+
+ assert result == {"error": ["preflight sentinel"]}
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "selected_name"),
+ [
+ ("ec2", "AWS_ACCESS_KEY_ID"),
+ ("ec2", "SSH_AUTH_SOCK"),
+ ("docker", "SSH_AUTH_SOCK"),
+ ("modal", "MODAL_SERVER_URL"),
+ ("gke", "CLOUDSDK_ACTIVE_CONFIG_NAME"),
+ ("gke", "CLOUDSDK_CORE_PROJECT"),
+ ("ack", "KUBECONFIG"),
+ ("ack", "KUBERNETES_SERVICE_HOST"),
+ ("ack", "KUBERNETES_SERVICE_PORT"),
+ ("ack", "DOCKER_CONFIG"),
+ ("openshift", "KUBECONFIG"),
+ ("novita", "NOVITA_SANDBOX_URL"),
+ ("singularity", "APPTAINER_AUTHFILE"),
+ ("tensorlake", "TENSORLAKE_ORGANIZATION_ID"),
+ ("cua-cloud", "CUA_CLIENT_SECRET"),
+ ("blaxel", "BL_API_KEY"),
+ ("opensandbox", "OPENSANDBOX_DOMAIN"),
+ ("beam", "BEAM_TOKEN"),
+ ("beam", "API_HOST"),
+ ("skypilot", "SKYPILOT_API_SERVER_ENDPOINT"),
+ ("skypilot", "DOCKER_CONFIG"),
+ ("hf-sandbox", "HF_TOKEN_PATH"),
+ ("hyperbrowser", "HYPERBROWSER_API_KEY"),
+ ("hyperbrowser", "DOCKER_HOST"),
+ ("wandb", "NETRC"),
+ ("vercel", "VERCEL_OIDC_TOKEN"),
+ ],
+)
+def test_new_harbor_022_backend_environment_is_selected_without_cross_backend_leakage(
+ monkeypatch: pytest.MonkeyPatch,
+ env_mode: str,
+ selected_name: str,
+) -> None:
+ host_environment = {
+ "PATH": "/usr/bin",
+ "HOME": "/home/test",
+ selected_name: "selected-value",
+ "E2B_API_KEY": "unselected-value",
+ }
+ monkeypatch.setattr(runner.os, "environ", host_environment)
+ provider = _provider()
+
+ environment = runner._harbor_subprocess_environment(
+ env_mode=env_mode,
+ provider=provider,
+ configured_runtime_env={},
+ provider_env=runner._provider_environment(provider),
+ )
+
+ assert environment[selected_name] == "selected-value"
+ assert "E2B_API_KEY" not in environment
+
+
def test_daytona_subprocess_environment_preserves_jwt_auth_pair(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -1433,9 +2022,9 @@ def test_config_declared_substitution_is_resolved_without_leaking_source_variabl
monkeypatch.setattr(
runner.os,
"environ",
- {"PATH": "/usr/bin", "HOME": "/home/test", "HOST_AGENT_TOKEN": "runtime-key", "UNRELATED_SECRET": "no"},
+ {"PATH": "/usr/bin", "HOME": "/home/test", "HOST_AGENT_LABEL": "runtime-value", "UNRELATED_SECRET": "no"},
)
- configured_runtime_env, errors = runner._resolve_runtime_env({"AGENT_TOKEN": "${HOST_AGENT_TOKEN}"})
+ configured_runtime_env, errors = runner._resolve_runtime_env({"AGENT_LABEL": "${HOST_AGENT_LABEL}"})
provider = _provider()
environment = runner._harbor_subprocess_environment(
@@ -1446,8 +2035,8 @@ def test_config_declared_substitution_is_resolved_without_leaking_source_variabl
)
assert errors == []
- assert environment["AGENT_TOKEN"] == "runtime-key"
- assert "HOST_AGENT_TOKEN" not in environment
+ assert environment["AGENT_LABEL"] == "runtime-value"
+ assert "HOST_AGENT_LABEL" not in environment
assert "UNRELATED_SECRET" not in environment
@@ -1468,6 +2057,7 @@ def test_runtime_env_rejects_host_process_control_names() -> None:
"BASH_ENV",
"BASH_FUNC_hidden%%",
"NODE_OPTIONS",
+ "LLM_JUDGE_FALLBACK_MODELS",
"DOCKER_HOST",
"COMPOSE_FILE",
"HARBOR_HOME",
@@ -1475,6 +2065,7 @@ def test_runtime_env_rejects_host_process_control_names() -> None:
"HTTPS_PROXY",
"E2B_API_KEY",
"AWS_CONFIG_FILE",
+ "AWS_ENDPOINT_URL",
"AWS_PROFILE",
"AWS_WEB_IDENTITY_TOKEN_FILE",
)
@@ -1534,6 +2125,12 @@ def test_bedrock_subprocess_environment_keeps_only_explicit_aws_provider_values(
assert "OTHER_CLOUD_SECRET" not in environment
+def test_bedrock_host_credential_allowlist_is_staged_for_the_verifier() -> None:
+ expected = set(runner._BEDROCK_HOST_ENV_VARS) | {"AWS_REGION"}
+
+ assert set(_verifier_env_vars(dict.fromkeys(expected, "configured"))) == expected
+
+
def test_local_nvidia_provider_preserves_explicit_codex_credentials(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
diff --git a/tests/test_harbor_runner_status.py b/tests/test_harbor_runner_status.py
index 06d3de02..def24ea2 100644
--- a/tests/test_harbor_runner_status.py
+++ b/tests/test_harbor_runner_status.py
@@ -5,8 +5,11 @@
from __future__ import annotations
+import contextlib
import importlib
import json
+import os
+import signal
import sys
import threading
import time
@@ -17,6 +20,574 @@
from skillevaluator.tier3.harbor import collector, runner
+def test_published_execution_errors_are_redacted_bounded_and_counted() -> None:
+ github_token = "ghp_" + ("A" * 36)
+ errors = [f"launch failed with {github_token} " + ("x" * 65_536)]
+ errors.extend(f"distinct launch error {index}" for index in range(300))
+
+ published, total = runner._published_execution_errors(errors)
+
+ assert total == 301
+ assert len(published) == runner.PUBLISHED_EXECUTION_ERRORS_MAX
+ assert all(len(error) <= runner.PUBLISHED_EXECUTION_ERROR_MAX_CHARS for error in published)
+ assert github_token not in json.dumps(published)
+ assert "" in published[0]
+
+
+def test_published_execution_errors_bound_serialized_multibyte_sample() -> None:
+ github_token = "ghp_" + ("A" * 36)
+ errors = [
+ f"launch {index}: {github_token}\x1b[2J" + ("😀" * 2_048)
+ for index in range(runner.PUBLISHED_EXECUTION_ERRORS_MAX)
+ ]
+
+ published, total = runner._published_execution_errors(errors)
+ serialized = json.dumps(published, indent=2).encode("utf-8")
+
+ assert total == runner.PUBLISHED_EXECUTION_ERRORS_MAX
+ assert len(serialized) <= 64 * 1024
+ assert len(published) < total
+ assert github_token.encode() not in serialized
+ assert b"\\u001b" not in serialized
+ assert b"[2J" in serialized
+
+
+def test_published_execution_errors_count_distinct_details_beyond_truncation() -> None:
+ shared_prefix = "x" * (runner.PUBLISHED_EXECUTION_ERROR_MAX_CHARS * 2)
+
+ published, total = runner._published_execution_errors([f"{shared_prefix}-first", f"{shared_prefix}-second"])
+
+ assert total == 2
+ assert len(published) == 1
+
+
+def test_launch_error_overlay_preserves_hidden_collector_count_and_adds_new_diagnostics() -> None:
+ result = {
+ "execution_status": "failed",
+ "execution_errors": ["shared visible error"],
+ "execution_error_details_total": 302,
+ "execution_error_details_shown": 1,
+ "execution_error_details_truncated": True,
+ "error": ["shared visible error"],
+ }
+
+ runner._merge_launch_execution_errors(
+ result,
+ ["shared visible error", "new launch diagnostic"],
+ )
+
+ assert result["execution_status"] == "failed"
+ assert result["execution_errors"] == ["shared visible error", "new launch diagnostic"]
+ assert result["execution_error_details_total"] == 303
+ assert result["execution_error_details_shown"] == 2
+ assert result["execution_error_details_truncated"] is True
+ assert result["error"] == ["shared visible error"]
+
+
+@pytest.mark.parametrize(
+ ("declared_total", "expected_total", "expected_truncated"),
+ [
+ (True, 2, False),
+ (-1, 2, False),
+ (1.5, 2, False),
+ ((1 << 60), (1 << 53) - 1, True),
+ ],
+ ids=["bool", "negative", "float", "above-json-safe-integer"],
+)
+def test_launch_error_overlay_bounds_declared_integer_metadata(
+ declared_total: object,
+ expected_total: int,
+ expected_truncated: bool,
+) -> None:
+ result = {
+ "execution_errors": ["collector diagnostic"],
+ "execution_error_details_total": declared_total,
+ "execution_error_details_truncated": False,
+ }
+
+ runner._merge_launch_execution_errors(result, ["launch diagnostic"])
+
+ assert result["execution_error_details_total"] == expected_total
+ assert result["execution_error_details_shown"] == 2
+ assert result["execution_error_details_truncated"] is expected_truncated
+
+
+@pytest.mark.parametrize("return_code", [0, 7])
+def test_bounded_harbor_process_preserves_exit_and_combined_diagnostic_tail(
+ return_code: int,
+) -> None:
+ script = (
+ "import os; "
+ "os.write(1, b'old-prefix-' + b'x' * 256); "
+ "os.write(2, b'|useful-stderr-tail|'); "
+ f"raise SystemExit({return_code})"
+ )
+
+ result = runner._run_bounded_harbor_process(
+ [sys.executable, "-c", script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=5,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=64,
+ secret_values=set(),
+ )
+
+ assert result.returncode == return_code
+ assert result.output_exceeded is False
+ assert len(result.output_tail) <= 64
+ assert result.output_tail.endswith("|useful-stderr-tail|")
+
+
+def test_bounded_harbor_process_redacts_before_retaining_diagnostic_tail() -> None:
+ secret = "SYNTHETIC_SECRET_ABCDEF"
+ script = f"import os; os.write(2, {('X' + secret + '|' + secret + '!' * 8).encode()!r})"
+
+ result = runner._run_bounded_harbor_process(
+ [sys.executable, "-c", script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=5,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=43,
+ secret_values={secret},
+ )
+
+ assert result.returncode == 0
+ assert result.output_exceeded is False
+ assert secret not in result.output_tail
+ assert all(secret[index:] not in result.output_tail for index in range(len(secret) - 3))
+ assert result.output_tail.endswith("!" * 8)
+
+
+def test_bounded_harbor_process_drops_partial_secret_at_output_limit() -> None:
+ secret = "SYNTHETIC_SECRET_ABCDEF"
+ prefix = "ordinary-prefix|"
+ accepted_secret_prefix = secret[:-3]
+ script = f"import os,time; os.write(2, {(prefix + secret + '|overflow').encode()!r}); time.sleep(30)"
+
+ result = runner._run_bounded_harbor_process(
+ [sys.executable, "-c", script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=5,
+ max_output_bytes=len((prefix + accepted_secret_prefix).encode()),
+ diagnostic_tail_chars=128,
+ secret_values={secret},
+ )
+
+ assert result.output_exceeded is True
+ assert accepted_secret_prefix not in result.output_tail
+ assert all(secret[index:-3] not in result.output_tail for index in range(len(secret) - 6))
+
+
+def test_bounded_harbor_process_timeout_includes_blocked_stdin_delivery() -> None:
+ started = time.monotonic()
+
+ with pytest.raises(RuntimeError, match="timed out"):
+ runner._run_bounded_harbor_process(
+ [sys.executable, "-c", "import time; time.sleep(30)"],
+ env=dict(os.environ),
+ stdin_text="x" * (1024 * 1024),
+ timeout_seconds=0.1,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values=set(),
+ )
+
+ assert time.monotonic() - started < 3
+
+
+def test_bounded_harbor_process_contains_tree_after_stdin_delivery_error() -> None:
+ started = time.monotonic()
+
+ with pytest.raises(RuntimeError, match="stdin delivery failed"):
+ runner._run_bounded_harbor_process(
+ [sys.executable, "-c", "import time; time.sleep(30)"],
+ env=dict(os.environ),
+ stdin_text="\udcff",
+ timeout_seconds=5,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values=set(),
+ )
+
+ assert time.monotonic() - started < 3
+
+
+def test_bounded_harbor_process_timeout_preserves_redacted_diagnostic_tail() -> None:
+ secret = "synthetic-timeout-secret"
+ script = f"import os,time;os.write(2,{f'useful timeout diagnostic {secret}'.encode()!r});time.sleep(30)"
+
+ with pytest.raises(RuntimeError, match="timed out") as raised:
+ runner._run_bounded_harbor_process(
+ [sys.executable, "-c", script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=0.1,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values={secret},
+ )
+
+ detail = str(raised.value)
+ assert "useful timeout diagnostic" in detail
+ assert secret not in detail
+ assert "redacted" in detail.lower()
+
+
+def test_bounded_harbor_process_redacts_secret_created_by_timeout_message() -> None:
+ secret = "Harbor run timed out after 0.1 seconds"
+
+ with pytest.raises(runner._HarborRunTimeoutError) as raised:
+ runner._run_bounded_harbor_process(
+ [sys.executable, "-c", "import time; time.sleep(30)"],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=0.1,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values={secret},
+ )
+
+ detail = str(raised.value)
+ assert secret not in detail
+ assert "redacted" in detail.lower()
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX cleanup for the red-state fallback")
+@pytest.mark.parametrize("failed_start", [1, 2])
+def test_bounded_harbor_process_owns_cleanup_during_thread_start(
+ monkeypatch: pytest.MonkeyPatch,
+ failed_start: int,
+) -> None:
+ real_popen = runner.subprocess.Popen
+ processes: list[object] = []
+ started_threads: list[threading.Thread] = []
+ start_count = 0
+
+ def tracked_popen(*args: object, **kwargs: object) -> object:
+ process = real_popen(*args, **kwargs)
+ processes.append(process)
+ return process
+
+ real_thread_start = threading.Thread.start
+
+ def failing_thread_start(thread: threading.Thread) -> None:
+ nonlocal start_count
+ start_count += 1
+ if start_count == failed_start:
+ raise RuntimeError("synthetic thread exhaustion")
+ real_thread_start(thread)
+ started_threads.append(thread)
+
+ monkeypatch.setattr(runner.subprocess, "Popen", tracked_popen)
+ monkeypatch.setattr(threading.Thread, "start", failing_thread_start)
+
+ try:
+ with pytest.raises(RuntimeError, match="synthetic thread exhaustion"):
+ runner._run_bounded_harbor_process(
+ [sys.executable, "-c", "import time; time.sleep(30)"],
+ env=dict(os.environ),
+ stdin_text="payload",
+ timeout_seconds=5,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values=set(),
+ )
+
+ assert len(processes) == 1
+ assert processes[0].poll() is not None # type: ignore[attr-defined]
+ assert all(not thread.is_alive() for thread in started_threads)
+ finally:
+ for process in processes:
+ if process.poll() is None: # type: ignore[attr-defined]
+ with contextlib.suppress(ProcessLookupError):
+ os.killpg(process.pid, signal.SIGKILL) # type: ignore[attr-defined]
+ with contextlib.suppress(Exception):
+ process.wait(timeout=5) # type: ignore[attr-defined]
+ for stream_name in ("stdin", "stdout"):
+ stream = getattr(process, stream_name, None)
+ if stream is not None:
+ with contextlib.suppress(OSError):
+ stream.close()
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process-group verification")
+@pytest.mark.parametrize("failure_mode", ["output", "timeout"])
+def test_bounded_harbor_process_failure_reaps_descendants(
+ tmp_path: Path,
+ failure_mode: str,
+) -> None:
+ child_pid_path = tmp_path / f"{failure_mode}-child-pid"
+ child_marker = tmp_path / f"{failure_mode}-child-survived"
+ child_script = (
+ "import pathlib,signal,time;"
+ "signal.signal(signal.SIGTERM, signal.SIG_IGN);"
+ "time.sleep(1);"
+ f"pathlib.Path({str(child_marker)!r}).write_text('survived')"
+ )
+ parent_script = (
+ "import os,pathlib,subprocess,sys,time;"
+ f"child=subprocess.Popen([{sys.executable!r},'-c',{child_script!r}]);"
+ f"pathlib.Path({str(child_pid_path)!r}).write_text(str(child.pid));"
+ + ("os.write(1,b'a'*3000);os.write(2,b'b'*3000);" if failure_mode == "output" else "")
+ + "time.sleep(30)"
+ )
+
+ started = time.monotonic()
+ if failure_mode == "timeout":
+ with pytest.raises(RuntimeError, match="timed out"):
+ runner._run_bounded_harbor_process(
+ [sys.executable, "-c", parent_script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=1,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values=set(),
+ )
+ else:
+ result = runner._run_bounded_harbor_process(
+ [sys.executable, "-c", parent_script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=5,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values=set(),
+ )
+ assert result.output_exceeded is True
+ assert time.monotonic() - started < 3
+
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ deadline = time.monotonic() + 1
+ while time.monotonic() < deadline:
+ try:
+ os.kill(child_pid, 0)
+ except ProcessLookupError:
+ break
+ time.sleep(0.01)
+ else:
+ with contextlib.suppress(ProcessLookupError):
+ os.kill(child_pid, signal.SIGKILL)
+ pytest.fail("Harbor descendant survived process-tree containment")
+ time.sleep(1.05)
+ assert not child_marker.exists()
+
+
+@pytest.mark.skipif(os.name != "nt", reason="requires Windows taskkill tree verification")
+@pytest.mark.parametrize("failure_mode", ["output", "timeout"])
+def test_bounded_harbor_process_failure_reaps_windows_descendants(
+ tmp_path: Path,
+ failure_mode: str,
+) -> None:
+ import ctypes
+ from ctypes import wintypes
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
+ kernel32.OpenProcess.restype = wintypes.HANDLE
+ kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
+ kernel32.CloseHandle.restype = wintypes.BOOL
+
+ child_pid_path = tmp_path / f"{failure_mode}-windows-child-pid"
+ child_marker = tmp_path / f"{failure_mode}-windows-child-survived"
+ child_script = f"import pathlib,time;time.sleep(1);pathlib.Path({str(child_marker)!r}).write_text('survived')"
+ parent_script = (
+ "import os,pathlib,subprocess,sys,time;"
+ f"child=subprocess.Popen([{sys.executable!r},'-c',{child_script!r}]);"
+ f"pathlib.Path({str(child_pid_path)!r}).write_text(str(child.pid));"
+ + ("os.write(1,b'a'*3000);os.write(2,b'b'*3000);" if failure_mode == "output" else "")
+ + "time.sleep(30)"
+ )
+
+ if failure_mode == "timeout":
+ with pytest.raises(RuntimeError, match="timed out"):
+ runner._run_bounded_harbor_process(
+ [sys.executable, "-c", parent_script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=1,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values=set(),
+ )
+ else:
+ result = runner._run_bounded_harbor_process(
+ [sys.executable, "-c", parent_script],
+ env=dict(os.environ),
+ stdin_text=None,
+ timeout_seconds=5,
+ max_output_bytes=4096,
+ diagnostic_tail_chars=128,
+ secret_values=set(),
+ )
+ assert result.output_exceeded is True
+
+ child_pid = int(child_pid_path.read_text(encoding="ascii"))
+ process_query_limited_information = 0x1000
+ deadline = time.monotonic() + 1
+ while time.monotonic() < deadline:
+ handle = kernel32.OpenProcess(
+ process_query_limited_information,
+ False,
+ child_pid,
+ )
+ if not handle:
+ break
+ kernel32.CloseHandle(handle)
+ time.sleep(0.01)
+ else:
+ pytest.fail("Harbor descendant survived Windows process-tree containment")
+ time.sleep(1.05)
+ assert not child_marker.exists()
+
+
+def test_windows_tree_cleanup_uses_verified_system32_taskkill_despite_path_and_cwd(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ system_directory = tmp_path / "trusted" / "System32"
+ system_directory.mkdir(parents=True)
+ trusted_taskkill = system_directory / "taskkill.exe"
+ trusted_taskkill.write_bytes(b"trusted")
+ decoy_directory = tmp_path / "decoy"
+ decoy_directory.mkdir()
+ decoy_taskkill = decoy_directory / "taskkill.exe"
+ decoy_taskkill.write_bytes(b"decoy")
+ monkeypatch.chdir(decoy_directory)
+ monkeypatch.setenv("PATH", str(decoy_directory))
+ monkeypatch.setattr(runner, "_windows_system_directory", lambda: system_directory, raising=False)
+
+ class WindowsOs:
+ name = "nt"
+
+ class FakeProcess:
+ def __init__(self, *, pid: int, returncode: int | None) -> None:
+ self.pid = pid
+ self.returncode = returncode
+ self.killed = False
+
+ def poll(self) -> int | None:
+ return self.returncode
+
+ def kill(self) -> None:
+ self.killed = True
+ self.returncode = -9
+
+ def wait(self, timeout: float | None = None) -> int:
+ del timeout
+ if self.returncode is None:
+ self.returncode = -9
+ return self.returncode
+
+ launched: list[list[str]] = []
+ taskkill_process = FakeProcess(pid=99, returncode=0)
+
+ def fake_popen(command: list[str], **_kwargs: object) -> FakeProcess:
+ launched.append(command)
+ return taskkill_process
+
+ root_process = FakeProcess(pid=4242, returncode=None)
+ monkeypatch.setattr(runner, "os", WindowsOs())
+ monkeypatch.setattr(runner.subprocess, "Popen", fake_popen)
+
+ runner._terminate_harbor_process_tree(root_process) # type: ignore[arg-type]
+
+ assert len(launched) == 1
+ selected_taskkill = Path(launched[0][0])
+ assert selected_taskkill.is_absolute()
+ assert selected_taskkill.resolve() == trusted_taskkill.resolve()
+ assert selected_taskkill.resolve() != decoy_taskkill.resolve()
+
+
+def test_windows_tree_cleanup_fails_closed_when_taskkill_misses_an_exited_root(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ class WindowsOs:
+ name = "nt"
+
+ class FakeProcess:
+ def __init__(self, *, pid: int, returncode: int | None) -> None:
+ self.pid = pid
+ self.returncode = returncode
+ self.killed = False
+
+ def poll(self) -> int | None:
+ return self.returncode
+
+ def kill(self) -> None:
+ self.killed = True
+ self.returncode = -9
+
+ def wait(self, timeout: float | None = None) -> int:
+ del timeout
+ return int(self.returncode or 0)
+
+ taskkill_process = FakeProcess(pid=99, returncode=1)
+ root_process = FakeProcess(pid=4242, returncode=0)
+ monkeypatch.setattr(runner, "os", WindowsOs())
+ monkeypatch.setattr(runner, "_verified_windows_taskkill_path", lambda: Path("/trusted/taskkill.exe"))
+ monkeypatch.setattr(runner.subprocess, "Popen", lambda *_args, **_kwargs: taskkill_process)
+
+ with pytest.raises(RuntimeError, match="process-tree cleanup could not be confirmed"):
+ runner._terminate_harbor_process_tree(root_process) # type: ignore[arg-type]
+
+ assert root_process.killed is False
+
+
+@pytest.mark.parametrize("candidate_kind", ["missing", "directory"])
+def test_windows_tree_cleanup_rejects_unverified_system32_taskkill(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ candidate_kind: str,
+) -> None:
+ system_directory = tmp_path / "trusted" / "System32"
+ system_directory.mkdir(parents=True)
+ if candidate_kind == "directory":
+ (system_directory / "taskkill.exe").mkdir()
+ monkeypatch.setattr(runner, "_windows_system_directory", lambda: system_directory, raising=False)
+
+ class WindowsOs:
+ name = "nt"
+
+ class FakeProcess:
+ def __init__(self, *, pid: int, returncode: int | None) -> None:
+ self.pid = pid
+ self.returncode = returncode
+ self.killed = False
+
+ def poll(self) -> int | None:
+ return self.returncode
+
+ def kill(self) -> None:
+ self.killed = True
+ self.returncode = -9
+
+ def wait(self, timeout: float | None = None) -> int:
+ del timeout
+ if self.returncode is None:
+ self.returncode = -9
+ return self.returncode
+
+ launched: list[list[str]] = []
+
+ def fake_popen(command: list[str], **_kwargs: object) -> FakeProcess:
+ launched.append(command)
+ return FakeProcess(pid=99, returncode=0)
+
+ root_process = FakeProcess(pid=4242, returncode=None)
+ monkeypatch.setattr(runner, "os", WindowsOs())
+ monkeypatch.setattr(runner.subprocess, "Popen", fake_popen)
+
+ with pytest.raises(RuntimeError, match="process-tree cleanup could not be confirmed"):
+ runner._terminate_harbor_process_tree(root_process) # type: ignore[arg-type]
+
+ assert launched == []
+ assert root_process.killed is True
+
+
@pytest.mark.parametrize("configured_concurrency", [1, 3, 4])
def test_agent_pair_treats_concurrency_as_a_global_condition_cap(
monkeypatch: pytest.MonkeyPatch,
@@ -444,30 +1015,58 @@ def test_merge_attempt_jobs_rejects_nested_directory_link_like_reparse_point(tmp
def test_merge_attempt_jobs_preserves_regular_trial_artifacts(tmp_path: Path) -> None:
+ from datetime import UTC, datetime
+ from uuid import UUID
+
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trial.result import TrialResult
+ from harbor.viewer.scanner import JobScanner
+
job_dir = tmp_path / "attempt-001"
trial_dir = job_dir / "case-001__trial"
artifacts = trial_dir / "artifacts"
artifacts.mkdir(parents=True)
(artifacts / "output.txt").write_text("expected", encoding="utf-8")
- (job_dir / "result.json").write_text(
- json.dumps(
- {
- "n_total_trials": 1,
- "stats": {
- "n_trials": 1,
- "n_errors": 0,
- "evals": {
- "demo": {
- "n_trials": 1,
- "n_errors": 0,
- "reward_stats": {"reward": {"1.0": [trial_dir.name]}},
- }
- },
- },
- }
- ),
- encoding="utf-8",
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ trial_result = TrialResult.model_validate(
+ {
+ "id": UUID(int=2),
+ "task_name": "nvidia/skillevaluator-case-001",
+ "trial_name": trial_dir.name,
+ "trial_uri": trial_dir.as_uri(),
+ "task_id": {"path": str(job_dir / "task" / "case-001")},
+ "source": "with",
+ "task_checksum": "harbor-0.22-aggregate-fixture",
+ "config": {
+ "task": {"path": str(job_dir / "task" / "case-001"), "source": "with"},
+ "trial_name": trial_dir.name,
+ "trials_dir": str(job_dir),
+ },
+ "agent_info": {
+ "name": "opencode",
+ "version": "test",
+ "model_info": {"name": "test-model"},
+ },
+ "agent_result": {"n_input_tokens": 7, "n_cache_tokens": 2, "n_output_tokens": 3},
+ "verifier_result": {"rewards": {"reward": 1.0}},
+ "started_at": now,
+ "finished_at": now,
+ }
)
+ source_job_result = JobResult(
+ id=UUID(int=1),
+ started_at=now,
+ updated_at=now,
+ finished_at=now,
+ n_total_trials=1,
+ stats=JobStats.from_trial_results([trial_result], n_total_trials=1),
+ # Harbor 0.22 currently persists completed TrialResults in each trial
+ # directory while this root list can remain empty.
+ trial_results=[],
+ )
+ (job_dir / "result.json").write_text(source_job_result.model_dump_json(indent=2), encoding="utf-8")
+ (trial_dir / "result.json").write_text(trial_result.model_dump_json(indent=2), encoding="utf-8")
+ (trial_dir / "config.json").write_text(trial_result.config.model_dump_json(indent=2), encoding="utf-8")
aggregate_dir = tmp_path / "aggregate"
runner._merge_attempt_jobs([job_dir], aggregate_dir)
@@ -475,7 +1074,382 @@ def test_merge_attempt_jobs_preserves_regular_trial_artifacts(tmp_path: Path) ->
merged_trial = aggregate_dir / f"{job_dir.name}__{trial_dir.name}"
assert (merged_trial / "artifacts" / "output.txt").read_text(encoding="utf-8") == "expected"
merged_result = json.loads((aggregate_dir / "result.json").read_text(encoding="utf-8"))
- assert merged_result["stats"]["evals"]["demo"]["reward_stats"]["reward"]["1.0"] == [merged_trial.name]
+ reward_names = [
+ name
+ for eval_stats in merged_result["stats"]["evals"].values()
+ for name in eval_stats["reward_stats"]["reward"]["1.0"]
+ ]
+ assert reward_names == [merged_trial.name]
+ scanner = JobScanner(tmp_path)
+ parsed_result = scanner.get_job_result(aggregate_dir.name)
+ parsed_config = scanner.get_job_config(aggregate_dir.name)
+ assert parsed_result is not None
+ assert parsed_result.n_total_trials == 1
+ assert parsed_result.stats.n_completed_trials == 1
+ assert len(parsed_result.trial_results) == parsed_result.stats.n_completed_trials
+ assert scanner.list_trials(aggregate_dir.name) == [merged_trial.name]
+ parsed_trial = scanner.get_trial_result(aggregate_dir.name, merged_trial.name)
+ assert parsed_trial is not None
+ assert parsed_trial.trial_name == merged_trial.name
+ assert parsed_trial.trial_uri == merged_trial.as_uri()
+ assert parsed_trial.config.trial_name == merged_trial.name
+ assert parsed_trial.config.trials_dir == aggregate_dir
+ assert parsed_result.trial_results[0] == parsed_trial
+ assert parsed_config is not None
+ assert parsed_config.job_name == aggregate_dir.name
+
+
+def test_merge_attempt_jobs_normalizes_mixed_naive_and_aware_timestamps(tmp_path: Path) -> None:
+ from harbor.viewer.scanner import JobScanner
+
+ job_dir = tmp_path / "attempt-001"
+ _write_current_harbor_attempt(
+ job_dir,
+ trial_name="case-001__trial",
+ root_completed=1,
+ )
+ root_result_path = job_dir / "result.json"
+ root_result = json.loads(root_result_path.read_text(encoding="utf-8"))
+ root_result.update(
+ {
+ "started_at": "2026-08-25T19:00:00",
+ "updated_at": "2026-08-25T19:01:00",
+ "finished_at": "2026-08-25T19:01:00",
+ }
+ )
+ root_result_path.write_text(json.dumps(root_result, indent=2), encoding="utf-8")
+ aggregate_dir = tmp_path / "aggregate"
+
+ runner._merge_attempt_jobs([job_dir], aggregate_dir)
+
+ merged = JobScanner(tmp_path).get_job_result(aggregate_dir.name)
+ assert merged is not None
+ assert merged.started_at.tzinfo is not None
+ assert merged.updated_at.tzinfo is not None
+ assert merged.finished_at is not None
+ assert merged.finished_at.tzinfo is not None
+ assert merged.started_at <= merged.updated_at
+
+
+def _write_current_harbor_attempt(
+ job_dir: Path,
+ *,
+ trial_name: str,
+ root_completed: int,
+ root_total: int = 1,
+ result_id: int = 20,
+ task_selector: str = "case",
+ task_name: str | None = None,
+ reward: float = 1.0,
+) -> None:
+ from datetime import UTC, datetime
+ from uuid import UUID
+
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trial.result import TrialResult
+
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ trial_result = TrialResult.model_validate(
+ {
+ "id": UUID(int=result_id),
+ "task_name": task_name or f"nvidia/{trial_name}",
+ "trial_name": trial_name,
+ "trial_uri": trial_dir.as_uri(),
+ "task_id": {"path": str(job_dir / "task" / task_selector)},
+ "source": "with",
+ "task_checksum": "harbor-0.22-aggregate-fixture",
+ "config": {
+ "task": {"path": str(job_dir / "task" / task_selector), "source": "with"},
+ "trial_name": trial_name,
+ "trials_dir": str(job_dir),
+ },
+ "agent_info": {
+ "name": "opencode",
+ "version": "test",
+ "model_info": {"name": "test-model"},
+ },
+ "agent_result": {},
+ "verifier_result": {"rewards": {"reward": reward}},
+ "started_at": now,
+ "finished_at": now,
+ }
+ )
+ root_results = [trial_result] if root_completed else []
+ root_stats = (
+ JobStats.from_trial_results(root_results, n_total_trials=root_total)
+ if root_completed
+ else JobStats(n_pending_trials=root_total)
+ )
+ source_job_result = JobResult(
+ id=UUID(int=result_id + 1),
+ started_at=now,
+ updated_at=now,
+ finished_at=now if root_completed == root_total else None,
+ n_total_trials=root_total,
+ stats=root_stats,
+ trial_results=[],
+ )
+ (job_dir / "result.json").write_text(source_job_result.model_dump_json(indent=2), encoding="utf-8")
+ (trial_dir / "result.json").write_text(trial_result.model_dump_json(indent=2), encoding="utf-8")
+ (trial_dir / "config.json").write_text(trial_result.config.model_dump_json(indent=2), encoding="utf-8")
+
+
+def test_merge_attempt_jobs_accepts_completed_child_ahead_of_stale_root_stats(tmp_path: Path) -> None:
+ from harbor.viewer.scanner import JobScanner
+
+ job_dir = tmp_path / "demo-with-case-attempt001"
+ _write_current_harbor_attempt(
+ job_dir,
+ trial_name="case-001__trial",
+ root_completed=0,
+ )
+ aggregate_dir = tmp_path / "aggregate"
+
+ runner._merge_attempt_jobs([job_dir], aggregate_dir)
+
+ merged = JobScanner(tmp_path).get_job_result(aggregate_dir.name)
+ assert merged is not None
+ assert merged.stats.n_completed_trials == 1
+ assert merged.stats.n_pending_trials == 0
+ assert merged.finished_at is not None
+
+
+def test_merge_attempt_jobs_rejects_root_completed_count_without_valid_child(tmp_path: Path) -> None:
+ from datetime import UTC, datetime
+ from uuid import UUID
+
+ from harbor.models.job.result import JobResult, JobStats
+
+ job_dir = tmp_path / "demo-with-case-attempt001"
+ job_dir.mkdir()
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ result = JobResult(
+ id=UUID(int=30),
+ started_at=now,
+ updated_at=now,
+ finished_at=now,
+ n_total_trials=1,
+ stats=JobStats(n_completed_trials=1),
+ trial_results=[],
+ )
+ (job_dir / "result.json").write_text(result.model_dump_json(indent=2), encoding="utf-8")
+
+ with pytest.raises(ValueError, match="completed 1 trials but retained 0"):
+ runner._merge_attempt_jobs([job_dir], tmp_path / "aggregate")
+
+
+@pytest.mark.parametrize(
+ ("job_name", "trial_name"),
+ [
+ ("s" * 180 + "-attempt001", "case-" + "t" * 120),
+ ("技" * 70 + "-attempt001", "例" * 70),
+ ],
+)
+def test_merge_attempt_jobs_bounds_long_aggregate_trial_names_by_utf8_bytes(
+ tmp_path: Path,
+ job_name: str,
+ trial_name: str,
+) -> None:
+ from harbor.viewer.scanner import JobScanner
+
+ job_dir = tmp_path / job_name
+ _write_current_harbor_attempt(job_dir, trial_name=trial_name, root_completed=1)
+ aggregate_dir = tmp_path / "aggregate"
+
+ runner._merge_attempt_jobs([job_dir], aggregate_dir)
+
+ merged_names = JobScanner(tmp_path).list_trials(aggregate_dir.name)
+ assert len(merged_names) == 1
+ assert len(merged_names[0].encode("utf-8")) <= 224
+ assert "attempt001" in merged_names[0]
+
+
+def test_long_stop_on_pass_merge_carries_runner_attempt_ordinal_into_collection(tmp_path: Path) -> None:
+ selector = "selector-attempt9"
+ logical_entry_id = "logical-attempt7"
+ jobs_dir = tmp_path / "jobs"
+ job_name = f"{'s' * 170}-opencode-with-{selector}-attempt002"
+ job_dir = jobs_dir / job_name
+ _write_current_harbor_attempt(
+ job_dir,
+ trial_name=f"{selector}__AbCd123",
+ root_completed=1,
+ task_selector=selector,
+ task_name="publisher/display-attempt5",
+ reward=1.0,
+ )
+ runner._merge_attempt_jobs([job_dir], jobs_dir / "demo-opencode-with")
+
+ result = collector.collect_harbor_results(
+ skill_name="demo",
+ agents=["opencode"],
+ output_dir=tmp_path / "results",
+ jobs_dir=jobs_dir,
+ skip_baseline=True,
+ n_attempts=3,
+ stop_on_pass=True,
+ expected_cases=1,
+ expected_case_ids=[logical_entry_id],
+ case_id_by_task_selector={selector: logical_entry_id},
+ )
+
+ assert result["execution_status"] == "failed"
+ assert result["expected_attempts"] == 2
+ assert result["scored_attempts"] == 1
+ assert any("Missing scored attempts" in error for error in result["execution_errors"])
+
+
+def test_merge_attempt_jobs_long_name_digest_avoids_cross_source_collisions(tmp_path: Path) -> None:
+ job_name = "s" * 180 + "-attempt001"
+ trial_name = "case-" + "t" * 120
+ first = tmp_path / "first" / job_name
+ second = tmp_path / "second" / job_name
+ _write_current_harbor_attempt(first, trial_name=trial_name, root_completed=1, result_id=40)
+ _write_current_harbor_attempt(second, trial_name=trial_name, root_completed=1, result_id=50)
+ aggregate_dir = tmp_path / "aggregate"
+
+ runner._merge_attempt_jobs([first, second], aggregate_dir)
+
+ merged_names = sorted(path.name for path in aggregate_dir.iterdir() if path.is_dir())
+ assert len(merged_names) == 2
+ assert len(set(merged_names)) == 2
+ assert all(len(name.encode("utf-8")) <= 224 for name in merged_names)
+
+
+def test_merge_attempt_jobs_rebuilds_cancel_retry_token_and_cost_stats(tmp_path: Path) -> None:
+ from datetime import UTC, datetime
+ from uuid import UUID
+
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trial.result import TrialResult
+ from harbor.viewer.scanner import JobScanner
+
+ job_dir = tmp_path / "attempt-001"
+ trial_dir = job_dir / "case-001__cancelled"
+ trial_dir.mkdir(parents=True)
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ trial_result = TrialResult.model_validate(
+ {
+ "id": UUID(int=4),
+ "task_name": "nvidia/skillevaluator-case-001",
+ "trial_name": trial_dir.name,
+ "trial_uri": trial_dir.as_uri(),
+ "task_id": {"path": str(job_dir / "task" / "case-001")},
+ "source": "with",
+ "task_checksum": "harbor-0.22-cancelled-fixture",
+ "config": {
+ "task": {"path": str(job_dir / "task" / "case-001"), "source": "with"},
+ "trial_name": trial_dir.name,
+ "trials_dir": str(job_dir),
+ },
+ "agent_info": {
+ "name": "opencode",
+ "version": "test",
+ "model_info": {"name": "test-model"},
+ },
+ "agent_result": {
+ "n_input_tokens": 123,
+ "n_cache_tokens": 4,
+ "n_output_tokens": 5,
+ "cost_usd": 0.25,
+ },
+ "verifier_result": None,
+ "exception_info": {
+ "exception_type": "CancelledError",
+ "exception_message": "cancelled",
+ "exception_traceback": "",
+ "occurred_at": now,
+ },
+ "started_at": now,
+ "finished_at": now,
+ }
+ )
+ source_stats = JobStats.from_trial_results([trial_result], n_total_trials=1, n_retries=2)
+ source_job_result = JobResult(
+ id=UUID(int=3),
+ started_at=now,
+ updated_at=now,
+ finished_at=now,
+ n_total_trials=1,
+ stats=source_stats,
+ trial_results=[],
+ )
+ (job_dir / "result.json").write_text(source_job_result.model_dump_json(indent=2), encoding="utf-8")
+ (trial_dir / "result.json").write_text(trial_result.model_dump_json(indent=2), encoding="utf-8")
+ (trial_dir / "config.json").write_text(trial_result.config.model_dump_json(indent=2), encoding="utf-8")
+ aggregate_dir = tmp_path / "aggregate"
+
+ runner._merge_attempt_jobs([job_dir], aggregate_dir)
+
+ merged = JobScanner(tmp_path).get_job_result(aggregate_dir.name)
+ assert merged is not None
+ assert merged.stats.n_completed_trials == 1
+ assert merged.stats.n_errored_trials == 1
+ assert merged.stats.n_cancelled_trials == 1
+ assert merged.stats.n_retries == 2
+ assert merged.stats.n_input_tokens == 123
+ assert merged.stats.n_cache_tokens == 4
+ assert merged.stats.n_output_tokens == 5
+ assert merged.stats.cost_usd == 0.25
+ assert len(merged.trial_results) == 1
+
+
+@pytest.mark.parametrize("with_job_result", [True, False])
+def test_merge_attempt_jobs_preserves_config_only_interrupted_trials(
+ tmp_path: Path,
+ with_job_result: bool,
+) -> None:
+ from datetime import UTC, datetime
+ from uuid import UUID
+
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trial.config import TrialConfig
+ from harbor.viewer.scanner import JobScanner
+
+ job_dir = tmp_path / "attempt-001"
+ trial_dir = job_dir / "case-001__interrupted"
+ trial_dir.mkdir(parents=True)
+ trial_config = TrialConfig.model_validate(
+ {
+ "task": {"path": str(job_dir / "task" / "case-001"), "source": "with"},
+ "trial_name": trial_dir.name,
+ "trials_dir": str(job_dir),
+ }
+ )
+ (trial_dir / "config.json").write_text(trial_config.model_dump_json(indent=2), encoding="utf-8")
+ if with_job_result:
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ source_job_result = JobResult(
+ id=UUID(int=5),
+ started_at=now,
+ updated_at=now,
+ finished_at=now,
+ n_total_trials=1,
+ stats=JobStats(n_pending_trials=1),
+ trial_results=[],
+ )
+ (job_dir / "result.json").write_text(source_job_result.model_dump_json(indent=2), encoding="utf-8")
+ aggregate_dir = tmp_path / "aggregate"
+
+ runner._merge_attempt_jobs([job_dir], aggregate_dir)
+
+ scanner = JobScanner(tmp_path)
+ merged = scanner.get_job_result(aggregate_dir.name)
+ assert merged is not None
+ assert merged.n_total_trials == 1
+ assert merged.stats.n_completed_trials == 0
+ assert merged.stats.n_pending_trials == 1
+ assert merged.trial_results == []
+ assert merged.finished_at is None
+ merged_trial_name = f"{job_dir.name}__{trial_dir.name}"
+ assert scanner.list_trials(aggregate_dir.name) == [merged_trial_name]
+ assert scanner.get_trial_result(aggregate_dir.name, merged_trial_name) is None
+ rewritten_config = scanner.get_trial_config(aggregate_dir.name, merged_trial_name)
+ assert rewritten_config is not None
+ assert rewritten_config.trial_name == merged_trial_name
+ assert rewritten_config.trials_dir == aggregate_dir
def test_merge_attempt_jobs_ignores_tmpdir_inside_attempt_job(
@@ -600,6 +1574,307 @@ def _write_job_result(jobs_dir: Path, stats: dict[str, object], *, total: int =
)
+def test_run_harbor_fails_closed_with_redacted_tail_on_output_overflow(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ secret = "synthetic-runner-overflow-secret"
+ monkeypatch.setattr(
+ runner,
+ "_run_bounded_harbor_process",
+ lambda *_args, **_kwargs: runner._BoundedHarborProcessResult(
+ returncode=-9,
+ output_tail=f"useful failure tail containing {secret}",
+ output_exceeded=True,
+ ),
+ )
+
+ ok, detail = runner._run_harbor(
+ dataset=tmp_path / "dataset",
+ agent="opencode",
+ job_name="demo-opencode-with",
+ env_mode="docker",
+ model="nvidia/model",
+ jobs_dir=tmp_path,
+ run_env={"API_KEY": secret},
+ n_attempts=1,
+ n_concurrent=1,
+ timeout_multiplier=1.0,
+ override_cpus=None,
+ override_memory_mb=None,
+ override_storage_mb=None,
+ )
+
+ assert ok is False
+ assert "output exceeded" in detail
+ assert "useful failure tail" in detail
+ assert secret not in detail
+ assert "redacted" in detail.lower()
+
+
+@pytest.mark.parametrize(
+ "proxy_uri",
+ [
+ "https://runner-user:runner-password@proxy.invalid:8443",
+ "https://runner%2Duser:runner%2Dpassword@proxy.invalid:8443",
+ "runner-user:runner-password@proxy.invalid:8443",
+ ],
+ ids=["uri", "percent-encoded", "schemeless"],
+)
+def test_run_harbor_redacts_detached_proxy_components_and_persisted_launch_error(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ proxy_uri: str,
+) -> None:
+ exposed = ("runner-user", "runner-password")
+ captured_secret_values: set[str] = set()
+
+ def bounded_process(*_args: object, **kwargs: object) -> runner._BoundedHarborProcessResult:
+ captured_secret_values.update(kwargs["secret_values"]) # type: ignore[arg-type]
+ return runner._BoundedHarborProcessResult(
+ returncode=7,
+ output_tail=f"proxy rejected {exposed[0]} with {exposed[1]}",
+ output_exceeded=False,
+ )
+
+ monkeypatch.setattr(
+ runner,
+ "build_harbor_run_command",
+ lambda **_kwargs: [sys.executable, "-c", "pass"],
+ )
+ monkeypatch.setattr(runner, "_run_bounded_harbor_process", bounded_process)
+
+ ok, detail = runner._run_harbor(
+ dataset=tmp_path / "dataset",
+ agent="opencode",
+ job_name="demo-opencode-with",
+ env_mode="docker",
+ model="nvidia/model",
+ jobs_dir=tmp_path,
+ run_env={"HTTPS_PROXY": proxy_uri},
+ n_attempts=1,
+ n_concurrent=1,
+ timeout_multiplier=1.0,
+ override_cpus=None,
+ override_memory_mb=None,
+ override_storage_mb=None,
+ )
+
+ assert ok is False
+ assert set(exposed) <= captured_secret_values
+ assert all(value not in detail for value in exposed)
+
+ persisted = {"execution_status": "failed", "execution_errors": []}
+ runner._merge_launch_execution_errors(persisted, [f"launch failed: {detail}"])
+ persisted_path = tmp_path / "persisted-launch-error.json"
+ persisted_path.write_text(json.dumps(persisted), encoding="utf-8")
+ persisted_text = persisted_path.read_text(encoding="utf-8")
+ assert all(value not in persisted_text for value in exposed)
+
+
+def test_run_harbor_real_subprocess_redacts_percent_decoded_proxy_components(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ monkeypatch.setattr(
+ runner,
+ "build_harbor_run_command",
+ lambda **_kwargs: [
+ sys.executable,
+ "-c",
+ "import sys; sys.stderr.write('proxy rejected process-user with process-password\\n'); raise SystemExit(7)",
+ ],
+ )
+
+ ok, detail = runner._run_harbor(
+ dataset=tmp_path / "dataset",
+ agent="opencode",
+ job_name="demo-opencode-with",
+ env_mode="docker",
+ model="nvidia/model",
+ jobs_dir=tmp_path,
+ run_env={
+ "HTTPS_PROXY": "https://process%2Duser:process%2Dpassword@proxy.invalid:8443",
+ },
+ n_attempts=1,
+ n_concurrent=1,
+ timeout_multiplier=1.0,
+ override_cpus=None,
+ override_memory_mb=None,
+ override_storage_mb=None,
+ )
+
+ assert ok is False
+ assert "proxy rejected" in detail
+ assert "process-user" not in detail
+ assert "process-password" not in detail
+
+
+@pytest.mark.parametrize("failure_path", ["nonzero", "output-limit", "timeout"])
+def test_run_harbor_redacts_detached_proxy_components_on_failure_paths(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ failure_path: str,
+) -> None:
+ proxy_uri = "https://failure-user:failure-password@proxy.invalid:8443"
+ diagnostic = "proxy failure for failure-user with failure-password"
+
+ def bounded_process(*_args: object, **_kwargs: object) -> runner._BoundedHarborProcessResult:
+ if failure_path == "timeout":
+ raise runner._HarborRunTimeoutError(f"Harbor run timed out. Last output: {diagnostic}")
+ return runner._BoundedHarborProcessResult(
+ returncode=7 if failure_path == "nonzero" else -9,
+ output_tail=diagnostic,
+ output_exceeded=failure_path == "output-limit",
+ )
+
+ monkeypatch.setattr(
+ runner,
+ "build_harbor_run_command",
+ lambda **_kwargs: [sys.executable, "-c", "pass"],
+ )
+ monkeypatch.setattr(runner, "_run_bounded_harbor_process", bounded_process)
+
+ ok, detail = runner._run_harbor(
+ dataset=tmp_path / "dataset",
+ agent="opencode",
+ job_name="demo-opencode-with",
+ env_mode="docker",
+ model="nvidia/model",
+ jobs_dir=tmp_path,
+ run_env={"HTTPS_PROXY": proxy_uri},
+ n_attempts=1,
+ n_concurrent=1,
+ timeout_multiplier=1.0,
+ override_cpus=None,
+ override_memory_mb=None,
+ override_storage_mb=None,
+ )
+
+ assert ok is False
+ assert "failure-user" not in detail
+ assert "failure-password" not in detail
+
+
+def test_run_harbor_reports_timeout_after_bounded_runner_contains_tree(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ monkeypatch.setattr(
+ runner,
+ "_run_bounded_harbor_process",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(runner._HarborRunTimeoutError("Harbor run timed out")),
+ )
+
+ ok, detail = _run(monkeypatch, tmp_path)
+
+ assert ok is False
+ assert detail == "Harbor run timed out"
+
+
+@pytest.mark.parametrize(
+ ("secret", "bounded_result"),
+ [
+ (
+ "safety",
+ runner._BoundedHarborProcessResult(
+ returncode=-9,
+ output_tail="ordinary overflow tail",
+ output_exceeded=True,
+ ),
+ ),
+ (
+ "harbor run exited 7",
+ runner._BoundedHarborProcessResult(
+ returncode=7,
+ output_tail="",
+ output_exceeded=False,
+ ),
+ ),
+ (
+ "result.json",
+ runner._BoundedHarborProcessResult(
+ returncode=0,
+ output_tail="",
+ output_exceeded=False,
+ ),
+ ),
+ ],
+)
+def test_run_harbor_redacts_secrets_created_by_final_diagnostic_assembly(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ secret: str,
+ bounded_result: runner._BoundedHarborProcessResult,
+) -> None:
+ monkeypatch.setattr(
+ runner,
+ "build_harbor_run_command",
+ lambda **_kwargs: [sys.executable, "-c", "pass"],
+ )
+ monkeypatch.setattr(runner, "_run_bounded_harbor_process", lambda *_args, **_kwargs: bounded_result)
+
+ ok, detail = runner._run_harbor(
+ dataset=tmp_path / "dataset",
+ agent="opencode",
+ job_name="demo-opencode-with",
+ env_mode="docker",
+ model="nvidia/model",
+ jobs_dir=tmp_path,
+ run_env={"API_KEY": secret},
+ n_attempts=1,
+ n_concurrent=1,
+ timeout_multiplier=1.0,
+ override_cpus=None,
+ override_memory_mb=None,
+ override_storage_mb=None,
+ )
+
+ assert ok is False
+ assert secret not in detail
+ assert "redacted" in detail.lower()
+
+
+def test_run_harbor_uses_collision_safe_marker_for_synthesized_secret(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ synthesized_secret = ""
+ monkeypatch.setattr(
+ runner,
+ "build_harbor_run_command",
+ lambda **_kwargs: [sys.executable, "-c", "pass"],
+ )
+ monkeypatch.setattr(
+ runner,
+ "_run_bounded_harbor_process",
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(runner._HarborRunTimeoutError("Harbor run timed out")),
+ )
+
+ ok, detail = runner._run_harbor(
+ dataset=tmp_path / "dataset",
+ agent="opencode",
+ job_name="demo-opencode-with",
+ env_mode="docker",
+ model="nvidia/model",
+ jobs_dir=tmp_path,
+ run_env={
+ "FIRST_API_KEY": "Harbor run timed out",
+ "SECOND_API_KEY": synthesized_secret,
+ },
+ n_attempts=1,
+ n_concurrent=1,
+ timeout_multiplier=1.0,
+ override_cpus=None,
+ override_memory_mb=None,
+ override_storage_mb=None,
+ )
+
+ assert ok is False
+ assert "Harbor run timed out" not in detail
+ assert synthesized_secret not in detail
+
+
def _complete_stats(**overrides: object) -> dict[str, object]:
stats: dict[str, object] = {
"n_trials": 1,
diff --git a/tests/test_harbor_runtime_preflight.py b/tests/test_harbor_runtime_preflight.py
index 634dc5c8..2f9f76fa 100644
--- a/tests/test_harbor_runtime_preflight.py
+++ b/tests/test_harbor_runtime_preflight.py
@@ -249,6 +249,72 @@ def test_runtime_preflight_accepts_harbor_0132_unscored_agent_success(monkeypatc
assert result.ok is True
+def test_agent_only_validation_accepts_actual_harbor_022_single_step_result(tmp_path: Path) -> None:
+ from datetime import UTC, datetime
+ from uuid import UUID
+
+ from harbor.models.job.result import JobResult, JobStats
+ from harbor.models.trial.result import TrialResult
+
+ job_dir = tmp_path / "jobs" / "runtime-preflight-opencode"
+ trial_dir = job_dir / "case-001__attempt"
+ trial_dir.mkdir(parents=True)
+ now = datetime(2026, 8, 25, tzinfo=UTC)
+ trial_result = TrialResult.model_validate(
+ {
+ "id": UUID(int=2),
+ "task_name": "nvidia/skillevaluator-case-001",
+ "trial_name": trial_dir.name,
+ "trial_uri": trial_dir.as_uri(),
+ "task_id": {"path": str(job_dir / "task" / "case-001")},
+ "task_checksum": "harbor-0.22-agent-only-fixture",
+ "config": {
+ "task": {"path": str(job_dir / "task" / "case-001")},
+ "trial_name": trial_dir.name,
+ "verifier": {"disable": True},
+ },
+ "agent_info": {
+ "name": "opencode",
+ "version": "test",
+ "model_info": {"name": "test-model"},
+ },
+ "agent_result": {
+ "n_input_tokens": 100,
+ "n_cache_tokens": 20,
+ "n_output_tokens": 10,
+ },
+ "verifier_result": None,
+ "exception_info": None,
+ "started_at": now,
+ "finished_at": now,
+ "step_results": None,
+ }
+ )
+ job_result = JobResult(
+ id=UUID(int=1),
+ started_at=now,
+ updated_at=now,
+ finished_at=now,
+ n_total_trials=1,
+ stats=JobStats.from_trial_results([trial_result], n_total_trials=1),
+ trial_results=[trial_result],
+ )
+ result_path = job_dir / "result.json"
+ result_path.write_text(job_result.model_dump_json(indent=2), encoding="utf-8")
+ (trial_dir / "result.json").write_text(trial_result.model_dump_json(indent=2), encoding="utf-8")
+
+ persisted_job = JobResult.model_validate_json(result_path.read_text(encoding="utf-8"))
+ persisted_trial = TrialResult.model_validate_json((trial_dir / "result.json").read_text(encoding="utf-8"))
+
+ assert persisted_job.stats.n_completed_trials == 1
+ assert persisted_job.stats.n_errored_trials == 0
+ assert persisted_job.stats.n_input_tokens == 100
+ assert persisted_job.stats.n_cache_tokens == 20
+ assert persisted_job.stats.n_output_tokens == 10
+ assert persisted_trial.step_results is None
+ assert runtime_preflight.validate_harbor_agent_only_job_result(result_path, expected_trials=1) == (True, "")
+
+
def test_agent_only_validation_accepts_harbor_0132_unscored_multistep_agent_success(tmp_path: Path) -> None:
result_path = _write_harbor_0132_unscored_result(tmp_path / "jobs")
trial_result_path = result_path.parent / "case-001__attempt" / "result.json"
@@ -659,6 +725,32 @@ def test_task_timeout_plan_uses_largest_staged_timeout(tmp_path: Path) -> None:
assert runner._task_timeout_plan([root], 2.0) == 600.0
+@pytest.mark.parametrize(
+ "task_toml",
+ [
+ "[agent]\ntimeout_sec = 900.0\n",
+ "[verifier]\ntimeout_sec = 900.0\n",
+ "[environment]\nbuild_timeout_sec = 900.0\n",
+ '[[steps]]\nname = "one"\n[steps.agent]\ntimeout_sec = 900.0\n',
+ '[[steps]]\nname = "one"\n[steps.verifier]\ntimeout_sec = 900.0\n',
+ ],
+ ids=["agent", "verifier", "environment-build", "step-agent", "step-verifier"],
+)
+def test_task_timeout_plan_rejects_staged_timeout_product_overflow(
+ tmp_path: Path,
+ task_toml: str,
+) -> None:
+ from skillevaluator.tier3.evals_config import MAX_HARBOR_TIMEOUT_MULTIPLIER
+ from skillevaluator.tier3.harbor import runner
+
+ task = tmp_path / "tasks" / "case-1"
+ task.mkdir(parents=True)
+ (task / "task.toml").write_text(task_toml, encoding="utf-8")
+
+ with pytest.raises(ValueError, match="non-finite Harbor timeout"):
+ runner._task_timeout_plan([task.parent], MAX_HARBOR_TIMEOUT_MULTIPLIER)
+
+
def test_model_probe_delegates_to_shared_catalog_client_without_exposing_key(monkeypatch) -> None:
captured: dict[str, object] = {}
@@ -3587,6 +3679,10 @@ def fail_preflight(**kwargs):
assert result["execution_status"] == "failed"
assert result["execution_errors"] == ["opencode runtime preflight failed: 401 Unauthorized"]
+ assert result["error"] == result["execution_errors"][:1]
+ assert result["execution_error_details_total"] == 1
+ assert result["execution_error_details_shown"] == 1
+ assert result["execution_error_details_truncated"] is False
assert preflight_run_env["LLM_JUDGE_MODEL"] == "host-legacy"
assert preflight_run_env["SKILL_EVAL_JUDGE_MODEL"] == "host-legacy"
full_matrix.assert_not_called()
@@ -3604,6 +3700,8 @@ def fail_preflight(**kwargs):
"model catalog access does not verify runtime credentials for this endpoint"
}
persisted = json.loads(result_path.read_text(encoding="utf-8"))
+ assert result_path.stat().st_size <= 2 * 1024 * 1024
+ assert persisted["error"] == persisted["execution_errors"][:1]
assert persisted["run_config"] == result["run_config"]
run_config_path = Path(result["run_dir"]) / "run_config.json"
assert json.loads(run_config_path.read_text(encoding="utf-8")) == result["run_config"]
diff --git a/tests/test_harbor_runtime_skill_isolation.py b/tests/test_harbor_runtime_skill_isolation.py
index 07aae84f..589014b9 100644
--- a/tests/test_harbor_runtime_skill_isolation.py
+++ b/tests/test_harbor_runtime_skill_isolation.py
@@ -17,6 +17,14 @@
from types import SimpleNamespace
import pytest
+from harbor.models.task.config import TaskConfig, TaskOS
+from harbor.models.task.paths import TaskPaths
+from harbor.models.task.task import Task
+from harbor.models.task.verifier_mode import resolve_effective_verifier_env_config
+from harbor.models.trajectories import Trajectory
+from harbor.models.trial.paths import TrialPaths
+from harbor.trial.trial import Trial
+from harbor.verifier.verifier import Verifier
import skillevaluator.tier3.harbor.adapter as adapter_module
from skillevaluator.tier3.harbor.adapter import (
@@ -33,6 +41,8 @@
from skillevaluator.tier3.output_provenance import mark_generated_output_root
from skillevaluator.tier3.results_location import publish_latest_results
+_SYNTHETIC_CREDENTIAL_DEFAULT = "".join(("sk-", "synthetic-default-12345678")) # noqa: FLY002
+
def _write_runtime_skill(path: Path, package: str) -> Path:
path.mkdir(parents=True)
@@ -226,12 +236,13 @@ def test_native_tasks_project_runtime_skills_without_root_evals(tmp_path: Path)
assert "SKILL_EVAL_JUDGE_MODEL" not in task_config["environment"].get("env", {})
-@pytest.mark.parametrize("name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
-@pytest.mark.parametrize("grading_mode", ["default", "custom_only"])
-def test_native_tasks_preserve_authored_judge_model_in_verifier_env(
+@pytest.mark.parametrize(
+ "name",
+ ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL", "LLM_JUDGE_FALLBACK_MODELS"],
+)
+def test_native_custom_only_preserves_authored_judge_controls_in_verifier_env(
tmp_path: Path,
name: str,
- grading_mode: str,
) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
_write_minimal_native_task(target)
@@ -244,19 +255,99 @@ def test_native_tasks_preserve_authored_judge_model_in_verifier_env(
task = stage_native_harbor_tasks(
target,
- tmp_path / f"native-verifier-{grading_mode}-{name.lower()}",
- grading_mode=grading_mode,
- verifier_env={
- "SKILL_EVAL_LLM_PROVIDER": "${SKILL_EVAL_LLM_PROVIDER}",
- name: f"${{{name}}}",
- },
+ tmp_path / f"native-custom-only-verifier-{name.lower()}",
+ grading_mode="custom_only",
)[0]
task_config = tomllib.loads((task / "task.toml").read_text(encoding="utf-8"))
- expected = {name: "skill-model"}
- if grading_mode == "default":
- expected["SKILL_EVAL_LLM_PROVIDER"] = "${SKILL_EVAL_LLM_PROVIDER}"
- assert task_config["verifier"]["env"] == expected
+ assert task_config["verifier"]["env"] == {name: "skill-model"}
+
+
+@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
+@pytest.mark.parametrize("scope", ["task", "step"])
+@pytest.mark.parametrize(
+ "name",
+ ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL", "LLM_JUDGE_FALLBACK_MODELS"],
+)
+def test_native_standard_grading_rejects_task_owned_judge_controls(
+ tmp_path: Path,
+ grading_mode: str,
+ scope: str,
+ name: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ authored_config = (
+ f'\n[verifier.env]\n{name} = "task-selected-model"\n'
+ if scope == "task"
+ else f'\n[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\n{name} = "task-selected-model"\n'
+ )
+ task_toml.write_text(task_toml.read_text(encoding="utf-8") + authored_config, encoding="utf-8")
+ output_dir = tmp_path / f"native-standard-{grading_mode}-{scope}-{name.lower()}"
+
+ with pytest.raises(ValueError, match=r"standard grading.*environment control"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode=grading_mode,
+ verifier_env={name: f"${{{name}}}"},
+ )
+
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize("scope", ["task", "step"])
+@pytest.mark.parametrize(
+ "name",
+ [
+ "OPENAI_API_KEY",
+ "SKILL_EVAL_LLM_PROVIDER",
+ "LLM_JUDGE_MODEL",
+ "SKILL_EVAL_JUDGE_MODEL",
+ "LLM_JUDGE_FALLBACK_MODELS",
+ ],
+)
+def test_native_standard_grading_allows_exact_operator_staged_control_placeholders(
+ tmp_path: Path,
+ scope: str,
+ name: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ authored_config = (
+ f'\n[verifier.env]\n{name} = "${{{name}}}"\n'
+ if scope == "task"
+ else f'\n[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\n{name} = "${{{name}}}"\n'
+ )
+ task_toml.write_text(task_toml.read_text(encoding="utf-8") + authored_config, encoding="utf-8")
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / f"native-operator-placeholder-{scope}-{name.lower()}",
+ grading_mode="default",
+ verifier_env={name: f"${{{name}}}"},
+ )
+
+ config = TaskConfig.model_validate_toml((staged / "task.toml").read_text(encoding="utf-8"))
+ environment = config.verifier.env if scope == "task" else config.steps[0].verifier.env
+ assert environment[name] == f"${{{name}}}"
+
+
+def test_native_standard_grading_injects_operator_staged_judge_fallback(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-operator-fallback",
+ grading_mode="default",
+ verifier_env={"LLM_JUDGE_FALLBACK_MODELS": "${LLM_JUDGE_FALLBACK_MODELS}"},
+ )
+
+ config = TaskConfig.model_validate_toml((staged / "task.toml").read_text(encoding="utf-8"))
+ assert config.verifier.env["LLM_JUDGE_FALLBACK_MODELS"] == "${LLM_JUDGE_FALLBACK_MODELS}"
@pytest.mark.parametrize(
@@ -284,86 +375,321 @@ def test_native_tasks_reject_judge_model_controls_in_agent_environment(tmp_path:
stage_native_harbor_tasks(target, tmp_path / "native-agent-judge-control")
-def test_native_task_reuses_existing_exact_provider_placeholder_when_injecting(tmp_path: Path) -> None:
+@pytest.mark.parametrize("source_name", ["DOCKER_AUTH_CONFIG", "DAYTONA_API_KEY"])
+@pytest.mark.parametrize(
+ "authored_config",
+ [
+ '[environment.env]\nTASK_ALIAS = "${SOURCE_NAME}"\n',
+ '[solution.env]\nTASK_ALIAS = "${SOURCE_NAME}"\n',
+ '[verifier.env]\nTASK_ALIAS = "${SOURCE_NAME}"\n',
+ '[verifier.environment]\nworkdir = "/workspace"\n\n[verifier.environment.env]\nTASK_ALIAS = "${SOURCE_NAME}"\n',
+ '[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\nTASK_ALIAS = "${SOURCE_NAME}"\n',
+ '[[steps]]\nname = "step-one"\n\n[steps.verifier.environment]\nworkdir = "/workspace"\n\n'
+ '[steps.verifier.environment.env]\nTASK_ALIAS = "${SOURCE_NAME}"\n',
+ ],
+)
+def test_native_tasks_reject_unstaged_parent_env_templates_in_every_harbor_scope(
+ tmp_path: Path,
+ source_name: str,
+ authored_config: str,
+) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
_write_minimal_native_task(target)
task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
- content = task_toml.read_text(encoding="utf-8").replace(
- "[environment]",
- "[verifier]\ntimeout_sec = 180.0\n\n[verifier.env]\n"
- 'SKILL_EVAL_LLM_PROVIDER = "${SKILL_EVAL_LLM_PROVIDER}"\n\n[environment]',
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8") + "\n" + authored_config.replace("SOURCE_NAME", source_name),
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError, match="outside the evaluator-staged role boundary"):
+ stage_native_harbor_tasks(
+ target,
+ tmp_path / f"native-host-env-{source_name.lower()}",
+ grading_mode="custom_only",
+ )
+
+
+def test_native_tasks_allow_only_role_staged_host_env_templates(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8")
+ + "\n"
+ + '[environment.env]\nAGENT_ALIAS = "${AGENT_VALUE}"\n\n'
+ + '[solution.env]\nSOLUTION_ALIAS = "${AGENT_VALUE}"\n\n'
+ + '[verifier]\nenvironment_mode = "separate"\n\n'
+ + '[verifier.env]\nVERIFY_ALIAS = "${VERIFY_VALUE}"\n\n'
+ + '[verifier.environment]\nworkdir = "/workspace"\n\n'
+ + '[verifier.environment.env]\nVERIFY_ENV_ALIAS = "${VERIFY_VALUE}"\n\n'
+ + '[[steps]]\nname = "step-one"\n\n'
+ + '[steps.verifier.env]\nSTEP_VERIFY_ALIAS = "${VERIFY_VALUE}"\n\n'
+ + '[steps.verifier.environment]\nworkdir = "/workspace"\n\n'
+ + '[steps.verifier.environment.env]\nSTEP_ENV_ALIAS = "${VERIFY_VALUE}"\n',
+ encoding="utf-8",
)
- task_toml.write_text(content, encoding="utf-8")
task = stage_native_harbor_tasks(
target,
- tmp_path / "native-existing-provider-placeholder",
- verifier_env={
- "SKILL_EVAL_LLM_MODEL": "${SKILL_EVAL_LLM_MODEL}",
- "SKILL_EVAL_LLM_PROVIDER": "${SKILL_EVAL_LLM_PROVIDER}",
- },
+ tmp_path / "native-role-env",
+ grading_mode="custom_only",
+ runtime_env={"AGENT_VALUE": "${AGENT_VALUE}"},
+ verifier_env={"VERIFY_VALUE": "${VERIFY_VALUE}"},
)[0]
- task_config = tomllib.loads((task / "task.toml").read_text(encoding="utf-8"))
- assert task_config["verifier"]["env"] == {
- "SKILL_EVAL_LLM_MODEL": "${SKILL_EVAL_LLM_MODEL}",
- "SKILL_EVAL_LLM_PROVIDER": "${SKILL_EVAL_LLM_PROVIDER}",
- }
+ config = TaskConfig.model_validate_toml((task / "task.toml").read_text(encoding="utf-8"))
+ assert config.environment.env["AGENT_ALIAS"] == "${AGENT_VALUE}"
+ assert config.environment.env["AGENT_VALUE"] == "${AGENT_VALUE}"
+ assert config.solution.env == {"SOLUTION_ALIAS": "${AGENT_VALUE}"}
+ assert config.verifier.env == {"VERIFY_ALIAS": "${VERIFY_VALUE}"}
+ assert config.verifier.environment is not None
+ assert config.verifier.environment.env == {"VERIFY_ENV_ALIAS": "${VERIFY_VALUE}"}
+ assert config.steps is not None
+ assert config.steps[0].verifier.env == {"STEP_VERIFY_ALIAS": "${VERIFY_VALUE}"}
+ assert config.steps[0].verifier.environment is not None
+ assert config.steps[0].verifier.environment.env == {"STEP_ENV_ALIAS": "${VERIFY_VALUE}"}
+
+
+def test_native_tasks_allow_safe_literal_default_for_role_staged_host_env_template(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8") + '\n[environment.env]\nAGENT_MODE = "${AGENT_VALUE:-offline}"\n',
+ encoding="utf-8",
+ )
+
+ task = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-safe-env-default",
+ grading_mode="custom_only",
+ runtime_env={"AGENT_VALUE": "${AGENT_VALUE}"},
+ )[0]
+
+ config = TaskConfig.model_validate_toml((task / "task.toml").read_text(encoding="utf-8"))
+ assert config.environment.env["AGENT_MODE"] == "${AGENT_VALUE:-offline}"
-@pytest.mark.parametrize("name", ["LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"])
@pytest.mark.parametrize(
- ("authored_config", "expected_path"),
+ ("key", "template", "protected_fragment"),
[
(
- '[verifier]\ntimeout_sec = 180.0\n\n[verifier.environment]\nworkdir = "/workspace"\n\n'
- '[verifier.environment.env]\n{name} = "skill-model"\n',
- ("verifier", "environment", "env"),
- ),
- (
- '[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\n{name} = "skill-model"\n',
- ("steps", 0, "verifier", "env"),
- ),
- (
- '[[steps]]\nname = "step-one"\n\n[steps.verifier.environment]\nworkdir = "/workspace"\n\n'
- '[steps.verifier.environment.env]\n{name} = "skill-model"\n',
- ("steps", 0, "verifier", "environment", "env"),
+ "AGENT_ALIAS",
+ f"${{AGENT_VALUE:-{_SYNTHETIC_CREDENTIAL_DEFAULT}}}",
+ _SYNTHETIC_CREDENTIAL_DEFAULT,
),
+ ("AGENT_ALIAS", "${AGENT_VALUE:-${OTHER_SECRET}}", "OTHER_SECRET"),
+ ("API_TOKEN", "${AGENT_VALUE:-offline}", "offline"),
],
)
-def test_native_tasks_preserve_judge_model_controls_in_nested_verifier_envs(
+def test_native_tasks_reject_credential_bearing_or_nested_role_staged_template_defaults_without_echo(
+ tmp_path: Path,
+ key: str,
+ template: str,
+ protected_fragment: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8") + f'\n[environment.env]\n{key} = "{template}"\n',
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError, match="literal credential-bearing assignment") as error:
+ stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-unsafe-env-default",
+ grading_mode="custom_only",
+ runtime_env={"AGENT_VALUE": "${AGENT_VALUE}"},
+ )
+ assert protected_fragment not in str(error.value)
+
+
+def test_native_tasks_reject_unstaged_template_defaults_and_literal_credentials_without_echo(
tmp_path: Path,
- name: str,
- authored_config: str,
- expected_path: tuple[str | int, ...],
) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
_write_minimal_native_task(target)
task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
task_toml.write_text(
- task_toml.read_text(encoding="utf-8") + "\n" + authored_config.format(name=name),
+ task_toml.read_text(encoding="utf-8")
+ + '\n[environment.env]\nAGENT_ALIAS = "${DOCKER_AUTH_CONFIG:-fallback-secret}"\n',
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError) as default_error:
+ stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-env-default",
+ grading_mode="custom_only",
+ )
+ assert "fallback-secret" not in str(default_error.value)
+
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8").replace(
+ 'AGENT_ALIAS = "${DOCKER_AUTH_CONFIG:-fallback-secret}"',
+ 'PASSWORD = "literal-credential-value"',
+ ),
+ encoding="utf-8",
+ )
+ with pytest.raises(ValueError) as literal_error:
+ stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-env-literal",
+ grading_mode="custom_only",
+ )
+ assert "literal-credential-value" not in str(literal_error.value)
+
+
+@pytest.mark.parametrize("verifier_scope", ["task", "step"])
+@pytest.mark.parametrize("source_name", ["DOCKER_AUTH_CONFIG", "DAYTONA_API_KEY"])
+def test_native_tasks_reject_parent_env_interpolation_in_separate_verifier_compose(
+ tmp_path: Path,
+ verifier_scope: str,
+ source_name: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ if verifier_scope == "task":
+ authored_config = '[verifier]\nenvironment_mode = "separate"\n\n[verifier.environment]\n'
+ verifier_context = native_task / "tests"
+ else:
+ authored_config = (
+ '[[steps]]\nname = "step-one"\n\n'
+ '[steps.verifier]\nenvironment_mode = "separate"\n\n'
+ "[steps.verifier.environment]\n"
+ )
+ verifier_context = native_task / "steps" / "step-one" / "tests"
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8") + "\n" + authored_config,
+ encoding="utf-8",
+ )
+ verifier_context.mkdir(parents=True, exist_ok=True)
+ (verifier_context / "docker-compose.yaml").write_text(
+ f"services:\n main:\n depends_on: [helper]\n helper:\n image: busybox:${{{source_name}}}\n",
+ encoding="utf-8",
+ )
+
+ output_dir = tmp_path / f"native-{verifier_scope}-verifier-compose-{source_name.lower()}"
+ with pytest.raises(ValueError, match="undeclared interpolation variables"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode="custom_only",
+ )
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize("verifier_scope", ["task", "step"])
+def test_native_tasks_allow_staged_verifier_env_interpolation_in_separate_verifier_compose(
+ tmp_path: Path,
+ verifier_scope: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ if verifier_scope == "task":
+ authored_config = '[verifier]\nenvironment_mode = "separate"\n\n[verifier.environment]\n'
+ verifier_context = native_task / "tests"
+ staged_relative_context = Path("tests")
+ else:
+ authored_config = (
+ '[[steps]]\nname = "step-one"\n\n'
+ '[steps.verifier]\nenvironment_mode = "separate"\n\n'
+ "[steps.verifier.environment]\n"
+ )
+ verifier_context = native_task / "steps" / "step-one" / "tests"
+ staged_relative_context = Path("steps") / "step-one" / "tests"
+ task_toml.write_text(
+ task_toml.read_text(encoding="utf-8") + "\n" + authored_config,
encoding="utf-8",
)
+ verifier_context.mkdir(parents=True, exist_ok=True)
+ compose_text = (
+ "services:\n"
+ " main:\n"
+ " depends_on: [helper]\n"
+ " helper:\n"
+ ' image: "busybox:${VERIFY_IMAGE}"\n'
+ ' ports: ["18080:80"]\n'
+ )
+ (verifier_context / "docker-compose.yaml").write_text(compose_text, encoding="utf-8")
+ if verifier_scope == "step":
+ # Harbor 0.22 resolves an existing separate step verifier context as a
+ # complete overlay, so it must provide its own test script instead of
+ # falling back to the task-level tests/test.sh.
+ (verifier_context / "test.sh").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+
+ staged_task = stage_native_harbor_tasks(
+ target,
+ tmp_path / f"native-{verifier_scope}-verifier-compose-allowed",
+ grading_mode="custom_only",
+ verifier_env={"VERIFY_IMAGE": "${VERIFY_IMAGE}"},
+ )[0]
+
+ assert (verifier_context / "docker-compose.yaml").read_text(encoding="utf-8") == compose_text
+ staged_compose = (staged_task / staged_relative_context / "docker-compose.yaml").read_text(encoding="utf-8")
+ assert "${VERIFY_IMAGE}" in staged_compose
+ assert "ports:" not in staged_compose
+
+
+def test_native_task_reuses_existing_exact_provider_placeholder_when_injecting(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ content = task_toml.read_text(encoding="utf-8").replace(
+ "[environment]",
+ "[verifier]\ntimeout_sec = 180.0\n\n[verifier.env]\n"
+ 'SKILL_EVAL_LLM_PROVIDER = "${SKILL_EVAL_LLM_PROVIDER}"\n\n[environment]',
+ )
+ task_toml.write_text(content, encoding="utf-8")
task = stage_native_harbor_tasks(
target,
- tmp_path / "native-nested-verifier-control",
+ tmp_path / "native-existing-provider-placeholder",
verifier_env={
+ "SKILL_EVAL_LLM_MODEL": "${SKILL_EVAL_LLM_MODEL}",
"SKILL_EVAL_LLM_PROVIDER": "${SKILL_EVAL_LLM_PROVIDER}",
- "SKILL_EVAL_JUDGE_MODEL": "${SKILL_EVAL_JUDGE_MODEL}",
},
)[0]
- value: object = tomllib.loads((task / "task.toml").read_text(encoding="utf-8"))
- for segment in expected_path:
- value = value[segment] # type: ignore[index]
- assert value == {name: "skill-model"}
task_config = tomllib.loads((task / "task.toml").read_text(encoding="utf-8"))
assert task_config["verifier"]["env"] == {
+ "SKILL_EVAL_LLM_MODEL": "${SKILL_EVAL_LLM_MODEL}",
"SKILL_EVAL_LLM_PROVIDER": "${SKILL_EVAL_LLM_PROVIDER}",
}
+@pytest.mark.parametrize("scope", ["task", "step"])
+def test_native_standard_grading_preserves_harmless_verifier_environment(
+ tmp_path: Path,
+ scope: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ authored_config = (
+ '\n[verifier.env]\nTASK_FEATURE_FLAG = "enabled"\n'
+ if scope == "task"
+ else '\n[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\nTASK_FEATURE_FLAG = "enabled"\n'
+ )
+ task_toml.write_text(task_toml.read_text(encoding="utf-8") + authored_config, encoding="utf-8")
+
+ [task] = stage_native_harbor_tasks(
+ target,
+ tmp_path / f"native-harmless-verifier-environment-{scope}",
+ grading_mode="default",
+ )
+
+ config = TaskConfig.model_validate_toml((task / "task.toml").read_text(encoding="utf-8"))
+ environment = config.verifier.env if scope == "task" else config.steps[0].verifier.env
+ assert environment == {"TASK_FEATURE_FLAG": "enabled"}
+
+
def test_native_task_staging_uses_one_private_evals_snapshot(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -2024,45 +2350,756 @@ def test_native_tasks_pin_skills_dir_to_controlled_projection(tmp_path: Path) ->
staged_toml = (staged / "task.toml").read_text(encoding="utf-8")
assert 'skills_dir = "/workspace/skills"' in staged_toml
- assert '"BASH_ENV" = ""' in staged_toml
- assert '"CLAUDE_CODE_DISABLE_POLICY_SKILLS" = "1"' in staged_toml
+ staged_config = tomllib.loads(staged_toml)
+ assert staged_config["environment"]["env"]["BASH_ENV"] == ""
+ assert staged_config["environment"]["env"]["CLAUDE_CODE_DISABLE_POLICY_SKILLS"] == "1"
-@pytest.mark.parametrize("name", ["BASH_ENV", "BASH_FUNC_hidden%%"])
-def test_generated_tasks_reject_runtime_process_loader_environment(tmp_path: Path, name: str) -> None:
+def test_native_task_structural_mutation_supports_inline_tables_and_quoted_keys(tmp_path: Path) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ source = (
+ 'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n'
+ '[metadata]\nentry_id = "case-001"\n\n'
+ '[verifier]\nenv = { AUTHORED = "kept" }\n\n'
+ '[environment]\nenv = { AUTHORED = "kept" }\n"skills_dir" = "/workspace/skills"\n'
+ )
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
- with pytest.raises(ValueError, match="process loader"):
- generate_harbor_tasks(
- target,
- tmp_path / "runtime-loader-env",
- runtime_env={name: "/runtime/seed.sh"},
- )
+ staged = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-inline-toml",
+ runtime_env={"RUNTIME_VALUE": "${RUNTIME_VALUE}"},
+ verifier_env={"SKILL_EVAL_LLM_PROVIDER": "${SKILL_EVAL_LLM_PROVIDER}"},
+ pre_agent_setup=["echo ready"],
+ )[0]
+
+ rendered = (staged / "task.toml").read_text(encoding="utf-8")
+ TaskConfig.model_validate_toml(rendered)
+ config = tomllib.loads(rendered)
+ assert config["environment"]["skills_dir"] == "/workspace/skills"
+ assert config["environment"]["env"]["AUTHORED"] == "kept"
+ assert config["environment"]["env"]["RUNTIME_VALUE"] == "${RUNTIME_VALUE}"
+ assert config["verifier"]["env"] == {
+ "AUTHORED": "kept",
+ "SKILL_EVAL_LLM_PROVIDER": "${SKILL_EVAL_LLM_PROVIDER}",
+ }
+ assert "echo ready" in config["environment"]["healthcheck"]["command"]
-def test_native_baseline_rejects_aliased_authored_skill(tmp_path: Path) -> None:
+def test_native_task_inline_healthcheck_conflicts_with_pre_agent_setup(tmp_path: Path) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
- native_task = target / "evals" / "harbor" / "case-001"
- environment = native_task / "environment"
- alias = environment / "skills" / "alias"
- alias.mkdir(parents=True)
- (alias / "SKILL.md").write_text((target / "SKILL.md").read_text(encoding="utf-8"), encoding="utf-8")
- (native_task / "instruction.md").write_text("Run the native case.\n", encoding="utf-8")
- (native_task / "task.toml").write_text(
- 'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n'
- '[metadata]\nentry_id = "case-001"\n\n[environment]\n',
- encoding="utf-8",
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ source = task_toml.read_text(encoding="utf-8").replace(
+ "[environment]\n",
+ '[environment]\nhealthcheck = { command = "true" }\n',
)
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
- with pytest.raises(ValueError, match="unmanaged skill"):
- stage_native_harbor_tasks(target, tmp_path / "native-baseline-alias", with_skill=False)
+ with pytest.raises(ValueError, match="healthcheck"):
+ stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-inline-healthcheck",
+ grading_mode="custom_only",
+ pre_agent_setup=["echo ready"],
+ )
-def test_native_baseline_rejects_renamed_target_manifest_payload(tmp_path: Path) -> None:
+@pytest.mark.parametrize("task_source", ["generated", "native"])
+def test_direct_baseline_staging_rejects_pre_agent_setup_before_output_mutation(
+ tmp_path: Path,
+ task_source: str,
+) -> None:
_, target, _, _ = _write_projection_fixture(tmp_path)
- native_task = target / "evals" / "harbor" / "case-001"
- environment = native_task / "environment"
- environment.mkdir(parents=True)
+ if task_source == "native":
+ _write_minimal_native_task(target)
+ output_dir = tmp_path / f"{task_source}-baseline-setup"
+ stager = generate_harbor_tasks if task_source == "generated" else stage_native_harbor_tasks
+
+ with pytest.raises(ValueError, match=r"pre_agent_setup/setup_commands.*baseline"):
+ stager(
+ target,
+ output_dir,
+ with_skill=False,
+ pre_agent_setup=["echo ready"],
+ )
+
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize("healthcheck_scope", ["environment", "step"])
+def test_native_baseline_rejects_authored_agent_phase_healthchecks(
+ tmp_path: Path,
+ healthcheck_scope: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8")
+ if healthcheck_scope == "environment":
+ source = source.replace(
+ "[environment]\n",
+ '[environment]\nhealthcheck = { command = "test ! -e /workspace/skills/target-skill/SKILL.md" }\n',
+ )
+ else:
+ source += '\n[[steps]]\nname = "step-one"\n\n[steps.healthcheck]\ncommand = "true"\n'
+ config = TaskConfig.model_validate_toml(source)
+ if healthcheck_scope == "environment":
+ assert config.environment.healthcheck is not None
+ else:
+ assert config.steps is not None and config.steps[0].healthcheck is not None
+ task_toml.write_text(source, encoding="utf-8")
+ output_dir = tmp_path / f"native-baseline-{healthcheck_scope}-healthcheck"
+
+ with pytest.raises(ValueError, match=rf"{healthcheck_scope} healthcheck.*baseline"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ with_skill=False,
+ grading_mode="custom_only",
+ )
+
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize(
+ ("trajectory_scope", "relative_path"),
+ [
+ ("single-step", Path("trajectory.json")),
+ ("first-step", Path("steps/step-one/trajectory.json")),
+ ],
+)
+def test_native_baseline_rejects_effective_task_shipped_prior_trajectory_before_output_mutation(
+ tmp_path: Path,
+ trajectory_scope: str,
+ relative_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ if trajectory_scope == "first-step":
+ source = task_toml.read_text(encoding="utf-8") + '\n[[steps]]\nname = "step-one"\n'
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ step_dir = native_task / "steps" / "step-one"
+ step_dir.mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Continue from the supplied context.\n", encoding="utf-8")
+
+ prior = {
+ "schema_version": "ATIF-v1.7",
+ "session_id": "prior-session",
+ "trajectory_id": "prior-trajectory",
+ "agent": {"name": "codex", "version": "test"},
+ "steps": [
+ {
+ "step_id": 1,
+ "source": "agent",
+ "message": "Prior task-owned context",
+ "tool_calls": [],
+ }
+ ],
+ }
+ Trajectory.model_validate(prior)
+ trajectory_path = native_task / relative_path
+ trajectory_path.parent.mkdir(parents=True, exist_ok=True)
+ trajectory_path.write_text(json.dumps(prior), encoding="utf-8")
+
+ config = TaskConfig.model_validate_toml(task_toml.read_text(encoding="utf-8"))
+ paths = TaskPaths(native_task)
+ effective_path = paths.step_trajectory_path(config.steps[0].name) if config.steps else paths.trajectory_path
+ assert effective_path == trajectory_path
+
+ baseline_dir = tmp_path / f"native-baseline-{trajectory_scope}-trajectory"
+ with pytest.raises(ValueError, match=r"prior trajectory.*paired baseline"):
+ stage_native_harbor_tasks(
+ target,
+ baseline_dir,
+ with_skill=False,
+ )
+
+ assert not baseline_dir.exists()
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / f"native-with-skill-{trajectory_scope}-trajectory",
+ with_skill=True,
+ )
+ assert (staged / relative_path).read_text(encoding="utf-8") == json.dumps(prior)
+
+
+@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
+@pytest.mark.parametrize(
+ ("scope", "variable", "value"),
+ [
+ ("environment", "SKILL_EVAL_LLM_BASE_URL", "https://task-controlled.example/v1"),
+ ("verifier", "HARBOR_TESTS_DIR", "/workspace/task-controlled-tests"),
+ ("step-verifier", "BASH_ENV", "/workspace/task-controlled-bootstrap"),
+ ("environment", "PATH", "/workspace/task-controlled-bin:/usr/bin:/bin"),
+ ("verifier", "HTTPS_PROXY", "http://task-controlled-proxy:8080"),
+ ("step-verifier", "SSL_CERT_FILE", "/workspace/task-controlled-ca.pem"),
+ ("step-verifier", "SHELLOPTS", "xtrace"),
+ ],
+)
+def test_native_standard_grading_rejects_task_controlled_verifier_environment_surfaces(
+ tmp_path: Path,
+ grading_mode: str,
+ scope: str,
+ variable: str,
+ value: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ source = task_toml.read_text(encoding="utf-8")
+ assignment = f'{variable} = "{value}"\n'
+ if scope == "environment":
+ source = source.replace("[environment]\n", f"[environment.env]\n{assignment}")
+ elif scope == "verifier":
+ source += f"\n[verifier.env]\n{assignment}"
+ else:
+ source += f'\n[[steps]]\nname = "step-one"\n\n[steps.verifier.env]\n{assignment}'
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ output_dir = tmp_path / f"native-{grading_mode}-{scope}-env"
+
+ with pytest.raises(ValueError, match=r"standard grading.*environment control"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode=grading_mode,
+ )
+
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
+@pytest.mark.parametrize("collect_scope", ["task", "step"])
+def test_native_standard_grading_rejects_post_agent_collect_hooks(
+ tmp_path: Path,
+ grading_mode: str,
+ collect_scope: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ task_toml = target / "evals" / "harbor" / "case-001" / "task.toml"
+ source = task_toml.read_text(encoding="utf-8")
+ if collect_scope == "task":
+ source += '\n[[verifier.collect]]\ncommand = "cp /workspace/result /logs/agent/trajectory.json"\n'
+ else:
+ source += (
+ '\n[[steps]]\nname = "step-one"\n\n'
+ '[[steps.verifier.collect]]\ncommand = "cp /workspace/result /logs/agent/trajectory.json"\n'
+ )
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ output_dir = tmp_path / f"native-{grading_mode}-{collect_scope}-collect"
+
+ with pytest.raises(ValueError, match=r"collect hook.*standard grading"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode=grading_mode,
+ )
+
+ assert not output_dir.exists()
+
+
+def test_native_custom_only_preserves_task_and_step_collect_hooks(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8") + (
+ '\n[[verifier.collect]]\ncommand = "cp /workspace/task-state /logs/agent/task-state"\n\n'
+ '[[steps]]\nname = "step-one"\n\n'
+ '[[steps.verifier.collect]]\ncommand = "cp /workspace/step-state /logs/agent/step-state"\n'
+ )
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ tests_dir = native_task / "tests"
+ tests_dir.mkdir()
+ (tests_dir / "test.sh").write_text("#!/bin/sh\nprintf 1 > /logs/verifier/reward.txt\n", encoding="utf-8")
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-custom-only-collect",
+ grading_mode="custom_only",
+ )
+
+ config = TaskConfig.model_validate_toml((staged / "task.toml").read_text(encoding="utf-8"))
+ assert [hook.command for hook in config.verifier.collect] == ["cp /workspace/task-state /logs/agent/task-state"]
+ assert config.steps is not None
+ assert [hook.command for hook in config.steps[0].verifier.collect] == [
+ "cp /workspace/step-state /logs/agent/step-state"
+ ]
+
+
+def test_native_custom_only_accepts_step_local_test_scripts_without_top_level_tests(
+ tmp_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8") + '\n[[steps]]\nname = "step-one"\n'
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ step_dir = native_task / "steps" / "step-one"
+ step_dir.mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+ step_test = step_dir / "tests" / "test.sh"
+ step_test.parent.mkdir()
+ step_test.write_text("#!/bin/sh\nprintf 1 > /logs/verifier/reward.txt\n", encoding="utf-8")
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-custom-only-step-tests",
+ grading_mode="custom_only",
+ )
+
+ assert not (staged / "tests" / "test.sh").exists()
+ assert (staged / "steps" / "step-one" / "tests" / "test.sh").read_text(encoding="utf-8") == (
+ "#!/bin/sh\nprintf 1 > /logs/verifier/reward.txt\n"
+ )
+
+
+def test_native_custom_only_rejects_top_level_fallback_hidden_by_existing_separate_step_context(
+ tmp_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8") + (
+ '\n[[steps]]\nname = "step-one"\n\n[steps.verifier]\nenvironment_mode = "separate"\n'
+ )
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ top_level_tests = native_task / "tests"
+ top_level_tests.mkdir()
+ (top_level_tests / "test.sh").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+ step_tests = native_task / "steps" / "step-one" / "tests"
+ step_tests.mkdir(parents=True)
+ (step_tests.parent / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+ (step_tests / "Dockerfile").write_text("FROM python:3.12-slim\n", encoding="utf-8")
+
+ harbor_task = Task(native_task, disable_verification=True)
+ assert harbor_task.config.steps is not None
+ build_context = Trial._verifier_env_build_context(
+ SimpleNamespace(task=harbor_task),
+ harbor_task.config.steps[0],
+ )
+ assert build_context.resolve() == step_tests.resolve()
+ assert not (build_context / "test.sh").exists()
+ output_dir = tmp_path / "native-custom-only-testless-separate-step-context"
+
+ with pytest.raises(FileNotFoundError, match=r"Harbor-resolvable test script for every verifier pass"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode="custom_only",
+ )
+
+ assert not output_dir.exists()
+
+
+def test_native_custom_only_accepts_top_level_fallback_for_separate_step_without_step_context(
+ tmp_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8") + (
+ '\n[[steps]]\nname = "step-one"\n\n[steps.verifier]\nenvironment_mode = "separate"\n'
+ )
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ tests_dir = native_task / "tests"
+ tests_dir.mkdir()
+ authored_test = "#!/bin/sh\nprintf 1 > /logs/verifier/reward.txt\n"
+ (tests_dir / "test.sh").write_text(authored_test, encoding="utf-8")
+ step_dir = native_task / "steps" / "step-one"
+ step_dir.mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+
+ harbor_task = Task(native_task, disable_verification=True)
+ assert harbor_task.config.steps is not None
+ build_context = Trial._verifier_env_build_context(
+ SimpleNamespace(task=harbor_task),
+ harbor_task.config.steps[0],
+ )
+ assert build_context.resolve() == tests_dir.resolve()
+ assert (build_context / "test.sh").is_file()
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-custom-only-top-level-separate-fallback",
+ grading_mode="custom_only",
+ )
+
+ assert (staged / "tests" / "test.sh").read_text(encoding="utf-8") == authored_test
+ assert not (staged / "steps" / "step-one" / "tests").exists()
+
+
+def test_native_custom_only_native_test_preserves_authored_skill_evaluator_package(
+ tmp_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_tests = target / "evals" / "harbor" / "case-001" / "tests"
+ authored_payload = native_tests / "skill_evaluator"
+ authored_payload.mkdir(parents=True)
+ helper = authored_payload / "custom_helper.py"
+ helper.write_text("CUSTOM_VALUE = 7\n", encoding="utf-8")
+ (native_tests / "test.sh").write_text(
+ '#!/bin/sh\npython3 -c "from skill_evaluator.custom_helper import CUSTOM_VALUE; assert CUSTOM_VALUE == 7"\n',
+ encoding="utf-8",
+ )
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-custom-only-authored-payload",
+ grading_mode="custom_only",
+ )
+
+ staged_payload = staged / "tests" / "skill_evaluator"
+ assert (staged_payload / "custom_helper.py").read_text(encoding="utf-8") == "CUSTOM_VALUE = 7\n"
+ assert not (staged_payload / "eval.py").exists()
+ assert not (staged_payload / "log_converters.py").exists()
+ assert not (staged_payload / "custom_grader_runner.py").exists()
+
+
+def test_native_standard_grader_runs_from_replaced_isolated_payload_directory(
+ tmp_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_tests = target / "evals" / "harbor" / "case-001" / "tests"
+ native_tests.mkdir()
+ marker = tmp_path / "shadow-module-imported"
+ (native_tests / "ipaddress.py").write_text(
+ f"from pathlib import Path\nPath({str(marker)!r}).write_text('imported')\n",
+ encoding="utf-8",
+ )
+
+ [staged] = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-isolated-standard-verifier",
+ grading_mode="default",
+ )
+
+ evaluator_dir = staged / "tests" / "skill_evaluator"
+ assert (staged / "tests" / "ipaddress.py").is_file()
+ assert (evaluator_dir / "eval.py").is_file()
+ test_sh = (staged / "tests" / "test.sh").read_text(encoding="utf-8")
+ assert 'python3 -I "${evaluator_dir}/eval.py"' in test_sh
+
+ agent_logs = tmp_path / "isolated-agent-logs"
+ verifier_logs = tmp_path / "isolated-verifier-logs"
+ agent_logs.mkdir()
+ verifier_logs.mkdir()
+ (agent_logs / "trajectory.json").write_text(
+ json.dumps(
+ {
+ "schema_version": "ATIF-v1.7",
+ "session_id": "isolated-verifier",
+ "trajectory_id": "isolated-verifier",
+ "agent": {"name": "codex", "version": "test"},
+ "steps": [{"step_id": 1, "source": "agent", "message": "done", "tool_calls": []}],
+ }
+ ),
+ encoding="utf-8",
+ )
+ completed = subprocess.run(
+ [sys.executable, "-I", str(evaluator_dir / "eval.py")],
+ capture_output=True,
+ text=True,
+ env={
+ **os.environ,
+ "HARBOR_TESTS_DIR": str(staged / "tests"),
+ "HARBOR_LOGS_DIR": str(tmp_path),
+ "HARBOR_AGENT_LOGS_DIR": str(agent_logs),
+ "HARBOR_VERIFIER_DIR": str(verifier_logs),
+ },
+ check=False,
+ )
+
+ assert completed.returncode == 1
+ assert "Required LLM judging failed" in completed.stderr
+ assert not marker.exists()
+
+
+@pytest.mark.parametrize(
+ "overlay_relative",
+ [Path("setup.sh"), Path(".agents/skills/renamed-target/SKILL.md")],
+)
+def test_native_baseline_rejects_every_configured_step_workdir_projection(
+ tmp_path: Path,
+ overlay_relative: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8") + '\n[[steps]]\nname = "step-one"\n'
+ config = TaskConfig.model_validate_toml(source)
+ assert config.steps is not None and config.steps[0].name == "step-one"
+ task_toml.write_text(source, encoding="utf-8")
+ projected = native_task / "steps" / "step-one" / "workdir" / overlay_relative
+ projected.parent.mkdir(parents=True)
+ projected.write_text("authored step payload\n", encoding="utf-8")
+ output_dir = tmp_path / f"native-baseline-workdir-{overlay_relative.name}"
+
+ with pytest.raises(ValueError, match=r"step workdir.*baseline"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ with_skill=False,
+ grading_mode="custom_only",
+ )
+
+ assert not output_dir.exists()
+
+
+def test_native_with_skill_staging_preserves_authored_healthchecks_and_step_workdir(
+ tmp_path: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8").replace(
+ "[environment]\n",
+ '[environment]\nhealthcheck = { command = "true" }\n',
+ )
+ source += '\n[[steps]]\nname = "step-one"\n\n[steps.healthcheck]\ncommand = "true"\n'
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ step_dir = native_task / "steps" / "step-one"
+ step_dir.mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+ workdir_file = step_dir / "workdir" / "fixture.txt"
+ workdir_file.parent.mkdir()
+ workdir_file.write_text("with-skill fixture\n", encoding="utf-8")
+
+ staged = stage_native_harbor_tasks(
+ target,
+ tmp_path / "native-with-skill-authored-setup",
+ with_skill=True,
+ grading_mode="custom_only",
+ )[0]
+
+ staged_config = TaskConfig.model_validate_toml((staged / "task.toml").read_text(encoding="utf-8"))
+ assert staged_config.environment.healthcheck is not None
+ assert staged_config.steps is not None and staged_config.steps[0].healthcheck is not None
+ assert (staged / "steps" / "step-one" / "workdir" / "fixture.txt").read_text(encoding="utf-8") == (
+ "with-skill fixture\n"
+ )
+
+
+@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
+@pytest.mark.parametrize("overlay_relative", [Path("test.sh"), Path("README.md")])
+def test_native_standard_grading_rejects_any_nonempty_step_tests_overlay(
+ tmp_path: Path,
+ grading_mode: str,
+ overlay_relative: Path,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8") + '\n[[steps]]\nname = "step-one"\n'
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ (native_task / "tests").mkdir()
+ (native_task / "tests" / "test.sh").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+ step_dir = native_task / "steps" / "step-one"
+ step_dir.mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+ overlay = step_dir / "tests" / overlay_relative
+ overlay.parent.mkdir(parents=True)
+ overlay.write_text("authored verifier overlay\n", encoding="utf-8")
+
+ harbor_task = Task(native_task, disable_verification=True)
+ verifier = Verifier(
+ harbor_task,
+ TrialPaths(tmp_path / "trial"),
+ SimpleNamespace(os=TaskOS.LINUX),
+ step_name="step-one",
+ )
+ test_sources, _selected_source, _selected_script = verifier._resolve_tests()
+ assert test_sources == [native_task / "tests", step_dir / "tests"]
+ output_dir = tmp_path / f"native-{grading_mode}-{overlay_relative.name}-overlay"
+
+ with pytest.raises(ValueError, match=r"step tests.*SkillEvaluator standard grading"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode=grading_mode,
+ )
+
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
+def test_native_standard_grading_allows_empty_step_tests_directory(
+ tmp_path: Path,
+ grading_mode: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ source = task_toml.read_text(encoding="utf-8") + '\n[[steps]]\nname = "step-one"\n'
+ TaskConfig.model_validate_toml(source)
+ task_toml.write_text(source, encoding="utf-8")
+ step_dir = native_task / "steps" / "step-one"
+ (step_dir / "tests").mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+
+ staged = stage_native_harbor_tasks(
+ target,
+ tmp_path / f"native-{grading_mode}-empty-step-tests",
+ grading_mode=grading_mode,
+ )[0]
+
+ assert (staged / "steps" / "step-one" / "tests").is_dir()
+ assert not any((staged / "steps" / "step-one" / "tests").iterdir())
+ assert (staged / "tests" / "test.sh").is_file()
+
+
+@pytest.mark.parametrize("grading_mode", ["default", "default_plus_custom"])
+@pytest.mark.parametrize("verifier_scope", ["task-explicit", "task-implicit", "step-explicit", "step-implicit"])
+def test_native_standard_grading_rejects_every_effective_separate_verifier_context(
+ tmp_path: Path,
+ grading_mode: str,
+ verifier_scope: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ header = 'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n[metadata]\nentry_id = "case-001"\n\n'
+ if verifier_scope == "task-explicit":
+ source = header + '[verifier]\nenvironment_mode = "separate"\n\n[environment]\n'
+ elif verifier_scope == "task-implicit":
+ source = header + '[verifier.environment]\nworkdir = "/workspace"\n\n[environment]\n'
+ elif verifier_scope == "step-explicit":
+ source = (
+ header
+ + '[[steps]]\nname = "step-one"\n\n[steps.verifier]\nenvironment_mode = "separate"\n\n[environment]\n'
+ )
+ else:
+ source = (
+ header
+ + '[[steps]]\nname = "step-one"\n\n[steps.verifier.environment]\nworkdir = "/workspace"\n\n[environment]\n'
+ )
+ config = TaskConfig.model_validate_toml(source)
+ step = config.steps[0] if config.steps else None
+ assert resolve_effective_verifier_env_config(config, step) is not None
+ task_toml.write_text(source, encoding="utf-8")
+ if step is not None:
+ step_dir = native_task / "steps" / step.name
+ step_dir.mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+ output_dir = tmp_path / f"native-{grading_mode}-{verifier_scope}"
+
+ with pytest.raises(ValueError, match=r"separate verifier.*SkillEvaluator standard grading"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode=grading_mode,
+ )
+
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize("windows_scope", ["agent", "task-verifier", "step-verifier"])
+def test_native_tasks_reject_every_effective_windows_execution_path(
+ tmp_path: Path,
+ windows_scope: str,
+) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ _write_minimal_native_task(target)
+ native_task = target / "evals" / "harbor" / "case-001"
+ task_toml = native_task / "task.toml"
+ header = 'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n[metadata]\nentry_id = "case-001"\n\n'
+ if windows_scope == "agent":
+ source = header + '[environment]\nos = "windows"\n'
+ elif windows_scope == "task-verifier":
+ source = header + '[verifier.environment]\nos = "windows"\n\n[environment]\nos = "linux"\n'
+ else:
+ source = (
+ header
+ + '[[steps]]\nname = "step-one"\n\n[steps.verifier.environment]\nos = "windows"\n\n'
+ + '[environment]\nos = "linux"\n'
+ )
+ config = TaskConfig.model_validate_toml(source)
+ step = config.steps[0] if config.steps else None
+ if windows_scope == "agent":
+ assert config.environment.os == TaskOS.WINDOWS
+ else:
+ effective_verifier = resolve_effective_verifier_env_config(config, step)
+ assert effective_verifier is not None and effective_verifier.os == TaskOS.WINDOWS
+ task_toml.write_text(source, encoding="utf-8")
+ if step is not None:
+ step_dir = native_task / "steps" / step.name
+ step_dir.mkdir(parents=True)
+ (step_dir / "instruction.md").write_text("Run step one.\n", encoding="utf-8")
+ output_dir = tmp_path / f"native-windows-{windows_scope}"
+
+ with pytest.raises(ValueError, match=r"Native Harbor.*Windows.*OS-aware"):
+ stage_native_harbor_tasks(
+ target,
+ output_dir,
+ grading_mode="custom_only",
+ )
+
+ assert not output_dir.exists()
+
+
+@pytest.mark.parametrize("name", ["BASH_ENV", "BASH_FUNC_hidden%%"])
+def test_generated_tasks_reject_runtime_process_loader_environment(tmp_path: Path, name: str) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+
+ with pytest.raises(ValueError, match="process loader"):
+ generate_harbor_tasks(
+ target,
+ tmp_path / "runtime-loader-env",
+ runtime_env={name: "/runtime/seed.sh"},
+ )
+
+
+def test_native_baseline_rejects_aliased_authored_skill(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ native_task = target / "evals" / "harbor" / "case-001"
+ environment = native_task / "environment"
+ alias = environment / "skills" / "alias"
+ alias.mkdir(parents=True)
+ (alias / "SKILL.md").write_text((target / "SKILL.md").read_text(encoding="utf-8"), encoding="utf-8")
+ (native_task / "instruction.md").write_text("Run the native case.\n", encoding="utf-8")
+ (native_task / "task.toml").write_text(
+ 'schema_version = "1.3"\n\n[task]\nname = "nvidia/case-001"\n\n'
+ '[metadata]\nentry_id = "case-001"\n\n[environment]\n',
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError, match="unmanaged skill"):
+ stage_native_harbor_tasks(target, tmp_path / "native-baseline-alias", with_skill=False)
+
+
+def test_native_baseline_rejects_renamed_target_manifest_payload(tmp_path: Path) -> None:
+ _, target, _, _ = _write_projection_fixture(tmp_path)
+ native_task = target / "evals" / "harbor" / "case-001"
+ environment = native_task / "environment"
+ environment.mkdir(parents=True)
(environment / "payload.txt").write_bytes((target / "SKILL.md").read_bytes())
(environment / "Dockerfile").write_text(
"FROM python:3.12-slim\nCOPY payload.txt /root/.agents/skills/alias/SKILL.md\n",
@@ -3504,6 +4541,45 @@ def record_read(path: Path, **kwargs: object) -> bytes:
assert not read_task_config
+def test_native_identity_helpers_preserve_numeric_zero_entry_id(tmp_path: Path) -> None:
+ native_root = tmp_path / "evals" / "harbor"
+ task_dir = native_root / "directory-selector"
+ task_dir.mkdir(parents=True)
+ (task_dir / "task.toml").write_text(
+ 'schema_version = "1.4"\n\n[task]\nname = "publisher/display-name"\n\n[metadata]\nentry_id = 0\n',
+ encoding="utf-8",
+ )
+
+ assert adapter_module._native_entry_id(task_dir) == "0"
+ assert adapter_module._native_projection_entry_ids(native_root) == ("0",)
+
+
+@pytest.mark.parametrize("entry_id", [True, 1.5], ids=["boolean", "noninteger-number"])
+def test_native_identity_helper_rejects_non_case_id_metadata_scalars(
+ tmp_path: Path,
+ entry_id: object,
+) -> None:
+ import toml
+
+ native_root = tmp_path / "evals" / "harbor"
+ task_dir = native_root / "directory-selector"
+ task_dir.mkdir(parents=True)
+ (task_dir / "task.toml").write_text(
+ toml.dumps(
+ {
+ "schema_version": "1.4",
+ "task": {"name": "publisher/display-name"},
+ "metadata": {"entry_id": entry_id},
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError, match="case id"):
+ adapter_module._native_entry_id(task_dir)
+ assert adapter_module._native_projection_entry_ids(native_root) == ("directory-selector",)
+
+
def test_link_or_reparse_check_accepts_missing_windows_file_attributes(tmp_path: Path) -> None:
path = tmp_path / "regular-file"
path.write_text("regular\n", encoding="utf-8")
diff --git a/tests/test_harbor_secure_docker_environment.py b/tests/test_harbor_secure_docker_environment.py
index d7daceac..6fb90719 100644
--- a/tests/test_harbor_secure_docker_environment.py
+++ b/tests/test_harbor_secure_docker_environment.py
@@ -4,29 +4,1557 @@
from __future__ import annotations
import asyncio
+import contextlib
+import errno
+import hashlib
+import io
import json
import os
+import random
+import shlex
+import shutil
import signal
import subprocess
+import sys
+import tarfile
+import time
+import tracemalloc
import uuid
from pathlib import Path
from types import MethodType, SimpleNamespace
import pytest
-from harbor.environments.base import ExecResult
+from harbor.environments.base import MAIN_SERVICE_NAME, ExecResult, ServiceOperationsUnsupportedError
+from harbor.environments.docker.docker import DockerEnvironment
+from harbor.models.task.config import EnvironmentConfig
+from harbor.models.trial.paths import TrialPaths
+from skillevaluator.tier3.harbor import stream_redaction as stream_redaction_module
from skillevaluator.tier3.harbor.adapter import generate_harbor_tasks
from skillevaluator.tier3.harbor.runner import build_harbor_run_command
from skillevaluator.tier3.harbor.secure_docker_environment import (
+ _REDACTION_SENTINEL_CANDIDATES,
+ NVIDIA_BUILD_STDIN_SENTINEL,
SECURE_DOCKER_ENV_IMPORT_PATH,
SkillEvaluatorDockerEnvironment,
SkillEvaluatorSecureDockerEnvironment,
+ _collision_safe_redaction_marker,
+ _compose_client_credential_values,
+ _compose_interpolation_names,
+ _redact,
_secure_exec_arguments,
+ _sidecar_environment_carriers,
+ _signal_process_tree,
+ _StreamingSecretRedactor,
)
+from skillevaluator.tier3.harbor.sensitive_stdin import NVIDIA_BUILD_KEY_STDIN_ENV
+from skillevaluator.tier3.harbor.stream_redaction import CommandOutputLimitError
_SENTINEL = "sentinel-never-visible-in-argv-or-files"
+def _marker_for(*secrets: str) -> str:
+ return _collision_safe_redaction_marker(secrets)
+
+
+def _initialized_docker_environment(
+ tmp_path: Path,
+ *,
+ environment_name: str = "secure-compose-test",
+ persistent_env: dict[str, str] | None = None,
+) -> SkillEvaluatorDockerEnvironment:
+ environment_dir = tmp_path / "environment"
+ environment_dir.mkdir(exist_ok=True)
+ (environment_dir / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
+ return SkillEvaluatorDockerEnvironment(
+ environment_dir=environment_dir,
+ environment_name=environment_name,
+ session_id="secure-compose-test",
+ trial_paths=TrialPaths(tmp_path / "trial"),
+ task_env_config=EnvironmentConfig(),
+ persistent_env=persistent_env,
+ )
+
+
+def _initialized_secure_docker_environment(
+ tmp_path: Path,
+ *,
+ persistent_env: dict[str, str] | None = None,
+) -> SkillEvaluatorSecureDockerEnvironment:
+ environment_dir = tmp_path / "secure-environment"
+ environment_dir.mkdir(exist_ok=True)
+ (environment_dir / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
+ return SkillEvaluatorSecureDockerEnvironment(
+ environment_dir=environment_dir,
+ environment_name="secure-compose-public-exec-test",
+ session_id="secure-compose-public-exec-test",
+ trial_paths=TrialPaths(tmp_path / "secure-trial"),
+ task_env_config=EnvironmentConfig(),
+ persistent_env=persistent_env,
+ )
+
+
+class _BufferedComposeProcess:
+ pid = 8841
+
+ def __init__(
+ self,
+ *,
+ stdout: bytes = b"buffered output",
+ stderr: bytes | None = None,
+ return_code: int = 0,
+ ) -> None:
+ self.returncode: int | None = return_code
+ self._stdout = stdout
+ self._stderr = stderr
+ self.stdout = _ChunkStream([stdout] if stdout else [])
+ self.stdin = _WritableStdin()
+ self.communicate_inputs: list[bytes | None] = []
+ self.wait_count = 0
+
+ async def communicate(self, **kwargs: bytes | None) -> tuple[bytes, bytes | None]:
+ assert set(kwargs) <= {"input"}
+ self.communicate_inputs.append(kwargs.get("input"))
+ return self._stdout, self._stderr
+
+ async def wait(self) -> int:
+ self.wait_count += 1
+ return self.returncode or 0
+
+ def terminate(self) -> None:
+ self.returncode = -signal.SIGTERM
+
+ def kill(self) -> None:
+ self.returncode = -signal.SIGKILL
+
+
+class _WritableStdin:
+ def __init__(self) -> None:
+ self.data = bytearray()
+ self.closed = False
+
+ def write(self, data: bytes) -> None:
+ self.data.extend(data)
+
+ async def drain(self) -> None:
+ return None
+
+ def close(self) -> None:
+ self.closed = True
+
+ async def wait_closed(self) -> None:
+ return None
+
+
+class _ChunkStream:
+ def __init__(self, chunks: list[bytes]) -> None:
+ self._chunks = iter(chunks)
+
+ def __aiter__(self) -> _ChunkStream:
+ return self
+
+ async def __anext__(self) -> bytes:
+ try:
+ return next(self._chunks)
+ except StopIteration:
+ raise StopAsyncIteration from None
+
+ async def read(self, _limit: int) -> bytes:
+ try:
+ return await self.__anext__()
+ except StopAsyncIteration:
+ return b""
+
+
+class _FeedableStream:
+ def __init__(self) -> None:
+ self._chunks: asyncio.Queue[bytes | None] = asyncio.Queue()
+
+ def feed_data(self, data: bytes) -> None:
+ self._chunks.put_nowait(data)
+
+ def feed_eof(self) -> None:
+ self._chunks.put_nowait(None)
+
+ async def read(self, _limit: int) -> bytes:
+ chunk = await self._chunks.get()
+ return b"" if chunk is None else chunk
+
+
+class _ExitAwaitingStream:
+ def __init__(self, process: _HangingComposeProcess) -> None:
+ self._process = process
+
+ async def read(self, _limit: int) -> bytes:
+ self._process.started.set()
+ await asyncio.shield(self._process.completion())
+ return b""
+
+
+class _HangingComposeProcess:
+ def __init__(self, *, pid: int) -> None:
+ self.pid = pid
+ self.returncode: int | None = None
+ self.started = asyncio.Event()
+ self.stdin = _WritableStdin()
+ self.stdout = _ExitAwaitingStream(self)
+ self._completion: asyncio.Future[int] | None = None
+
+ def completion(self) -> asyncio.Future[int]:
+ if self._completion is None:
+ self._completion = asyncio.get_running_loop().create_future()
+ return self._completion
+
+ def finish(self, return_code: int) -> None:
+ self.returncode = return_code
+ completion = self.completion()
+ if not completion.done():
+ completion.set_result(return_code)
+
+ async def wait(self) -> int:
+ return await asyncio.shield(self.completion())
+
+ async def communicate(self, **_kwargs: bytes | None) -> tuple[bytes, None]:
+ self.started.set()
+ await asyncio.shield(self.completion())
+ return b"", None
+
+ def terminate(self) -> None:
+ self.finish(-signal.SIGTERM)
+
+ def kill(self) -> None:
+ self.finish(-signal.SIGKILL)
+
+
+class _StreamedComposeProcess:
+ pid = 8842
+
+ def __init__(self, chunks: list[bytes], *, return_code: int = 0) -> None:
+ self.returncode: int | None = None
+ self._exit_code = return_code
+ self.stdout = _ChunkStream(chunks)
+ self.stdin = _WritableStdin()
+ self.wait_count = 0
+ self.terminate_count = 0
+ self.kill_count = 0
+
+ async def wait(self) -> int:
+ self.wait_count += 1
+ if self.returncode is None:
+ self.returncode = self._exit_code
+ return self.returncode
+
+ def terminate(self) -> None:
+ self.terminate_count += 1
+ self.returncode = -signal.SIGTERM
+
+ def kill(self) -> None:
+ self.kill_count += 1
+ self.returncode = -signal.SIGKILL
+
+
+class _BufferedAndStreamedComposeProcess(_StreamedComposeProcess):
+ def __init__(self, chunks: list[bytes], *, return_code: int = 0) -> None:
+ super().__init__(chunks, return_code=return_code)
+ self._buffered_stdout = b"".join(chunks)
+ self.communicate_inputs: list[bytes | None] = []
+
+ async def communicate(self, **kwargs: bytes | None) -> tuple[bytes, None]:
+ assert set(kwargs) <= {"input"}
+ self.communicate_inputs.append(kwargs.get("input"))
+ self.returncode = self._exit_code
+ return self._buffered_stdout, None
+
+
+class _CallbackBaseError(BaseException):
+ pass
+
+
+def test_docker_streaming_accepts_newline_free_output_larger_than_reader_limit(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ output_size = 200_000
+ callbacks: list[str] = []
+
+ async def on_output(text: str, stream: str) -> None:
+ assert stream == "stdout"
+ callbacks.append(text)
+
+ async def exercise() -> ExecResult:
+ process = await asyncio.create_subprocess_exec(
+ sys.executable,
+ "-c",
+ f"import sys; sys.stdout.buffer.write(b'x' * {output_size}); sys.stdout.buffer.flush()",
+ stdin=asyncio.subprocess.DEVNULL,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.STDOUT,
+ )
+ try:
+ return await environment._collect_streamed_output(
+ process,
+ timeout_sec=5,
+ on_output=on_output,
+ )
+ finally:
+ if process.returncode is None:
+ process.kill()
+ await process.wait()
+
+ result = asyncio.run(exercise())
+
+ assert result.return_code == 0
+ assert result.stdout == "x" * output_size
+ assert "".join(callbacks) == result.stdout
+ assert all(len(chunk.encode()) <= 64 * 1024 for chunk in callbacks)
+
+
+@pytest.mark.parametrize("with_callback", [False, True])
+def test_compose_output_limit_fails_closed_and_contains_process(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ with_callback: bool,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ overflow_value = b"synthetic-overflow-value"
+ process = _BufferedAndStreamedComposeProcess([b"12345678", overflow_value])
+ callback_chunks: list[str] = []
+ containment_calls: list[asyncio.subprocess.Process] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedAndStreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ async def contain(
+ contained_process: asyncio.subprocess.Process,
+ communication: asyncio.Task[object],
+ **_kwargs: object,
+ ) -> None:
+ containment_calls.append(contained_process)
+ with contextlib.suppress(BaseException):
+ await communication
+
+ monkeypatch.setattr(stream_redaction_module, "MAX_COMMAND_OUTPUT_BYTES", 8, raising=False)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(environment, "_contain_main_and_reap_compose", contain)
+
+ callback = on_output if with_callback else None
+ with pytest.raises(CommandOutputLimitError, match=r"Command output exceeded the 8-byte safety limit") as caught:
+ asyncio.run(
+ environment._run_docker_compose_command(
+ ["version"],
+ check=False,
+ on_output=callback,
+ )
+ )
+
+ assert containment_calls == [process]
+ assert process.returncode is not None
+ assert len("".join(callback_chunks).encode()) <= 8
+ assert overflow_value.decode() not in "".join(callback_chunks)
+ assert overflow_value.decode() not in str(caught.value)
+
+
+def test_compose_bounded_path_uses_devnull_without_stdin_or_callback(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _BufferedComposeProcess()
+ captured: dict[str, object] = {}
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ captured.update(kwargs)
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ result = asyncio.run(environment._run_docker_compose_command(["version"], check=False))
+
+ assert captured["stdin"] == asyncio.subprocess.DEVNULL
+ assert process.communicate_inputs == []
+ assert result == ExecResult(stdout="buffered output", stderr=None, return_code=0)
+
+
+@pytest.mark.parametrize("stdin_data", [b"", b"\x00tar\xffpayload\nwith spaces\x00"])
+def test_compose_stdin_reaches_bounded_subprocess_byte_for_byte(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ stdin_data: bytes,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _BufferedComposeProcess()
+ captured: dict[str, object] = {}
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ captured.update(kwargs)
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ result = asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "-T", "main", "tar", "-xf", "-"],
+ check=False,
+ stdin_data=stdin_data,
+ )
+ )
+
+ assert captured["stdin"] == asyncio.subprocess.PIPE
+ assert process.communicate_inputs == []
+ assert bytes(process.stdin.data) == stdin_data
+ assert result.return_code == 0
+
+
+def test_compose_stream_callback_writes_and_closes_stdin(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([b"archive received\n"])
+ captured: dict[str, object] = {}
+ callback_chunks: list[str] = []
+ stdin_data = b"\x00streamed\xffarchive\n"
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _StreamedComposeProcess:
+ captured.update(kwargs)
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ result = asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "-T", "main", "tar", "-xf", "-"],
+ check=False,
+ stdin_data=stdin_data,
+ on_output=on_output,
+ )
+ )
+
+ assert captured["stdin"] == asyncio.subprocess.PIPE
+ assert bytes(process.stdin.data) == stdin_data
+ assert process.stdin.closed is True
+ assert callback_chunks == ["archive received\n"]
+ assert result.stdout == "archive received\n"
+
+
+def test_compose_stream_callback_receives_merged_complete_redacted_output(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persistent_secret = "persistent-secret-value"
+ scoped_secret = "scoped-secret-value"
+ per_call_secret = "per-call-secret-value"
+ environment = _initialized_docker_environment(
+ tmp_path,
+ persistent_env={"PERSISTENT_TOKEN": persistent_secret},
+ )
+ process = _StreamedComposeProcess(
+ [
+ f"stdout {persistent_secret}\n".encode(),
+ f"stderr {scoped_secret}\n".encode(),
+ f"tail {per_call_secret}\n".encode(),
+ ]
+ )
+ captured: dict[str, object] = {}
+ callbacks: list[tuple[str, str]] = []
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _StreamedComposeProcess:
+ captured.update(kwargs)
+ return process
+
+ async def on_output(text: str, stream: str) -> None:
+ callbacks.append((text, stream))
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_exec_env({"SCOPED_TOKEN": scoped_secret}):
+ merged_environment = environment._merge_env({"PER_CALL_TOKEN": per_call_secret})
+ assert merged_environment is not None
+ return await environment._run_docker_compose_command(
+ [
+ "exec",
+ "-e",
+ "PERSISTENT_TOKEN",
+ "-e",
+ "SCOPED_TOKEN",
+ "-e",
+ "PER_CALL_TOKEN",
+ "main",
+ "emit-output",
+ ],
+ check=False,
+ on_output=on_output,
+ env_overrides=merged_environment,
+ )
+
+ result = asyncio.run(exercise())
+
+ assert captured["stderr"] == asyncio.subprocess.STDOUT
+ child_environment = captured["env"]
+ assert isinstance(child_environment, dict)
+ assert child_environment["PERSISTENT_TOKEN"] == persistent_secret
+ assert child_environment["SCOPED_TOKEN"] == scoped_secret
+ assert child_environment["PER_CALL_TOKEN"] == per_call_secret
+ assert {stream for _text, stream in callbacks} == {"stdout"}
+ assert "".join(text for text, _stream in callbacks) == result.stdout
+ marker = _marker_for(persistent_secret, scoped_secret, per_call_secret)
+ assert result.stdout == f"stdout {marker}\nstderr {marker}\ntail {marker}\n"
+ assert result.stderr is None
+ assert result.return_code == 0
+
+
+@pytest.mark.parametrize(
+ ("environment_name", "credential_value"),
+ [
+ ("DOCKER_HOST", "tcp://compose-user:compose-password@docker.invalid:2376"),
+ ("HTTPS_PROXY", "https://proxy-user:proxy-password@proxy.invalid:8443"),
+ ("HTTPS_PROXY", "proxy-user:proxy-password@proxy.invalid:8443"),
+ (
+ "DOCKER_AUTH_CONFIG",
+ '{"auths":{"registry.invalid":{"auth":"compose-registry-credential"}}}',
+ ),
+ ],
+)
+def test_compose_redacts_host_client_credentials_from_callback_and_result(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ environment_name: str,
+ credential_value: str,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([f"client failure: {credential_value}\n".encode()])
+ callbacks: list[str] = []
+ captured_environment: dict[str, str] = {}
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _StreamedComposeProcess:
+ captured_environment.update(kwargs["env"])
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callbacks.append(text)
+
+ monkeypatch.setenv(environment_name, credential_value)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ result = asyncio.run(
+ environment._run_docker_compose_command(
+ ["version"],
+ check=False,
+ on_output=on_output,
+ )
+ )
+
+ callback_output = "".join(callbacks)
+ assert captured_environment[environment_name] == credential_value
+ assert callback_output == result.stdout
+ assert credential_value not in callback_output
+ assert credential_value not in (result.stdout or "")
+ compose_credential_values = _compose_client_credential_values({environment_name: credential_value})
+ assert _marker_for(*compose_credential_values) in callback_output
+
+
+@pytest.mark.parametrize(
+ ("proxy_uri", "diagnostic", "secrets"),
+ [
+ (
+ "https://proxy-user:proxy-password@proxy.invalid:8443",
+ "proxy auth failed for proxy-user with proxy-password\n",
+ ("proxy-user", "proxy-password"),
+ ),
+ (
+ "https://proxy%2Duser:proxy%2Dpassword@proxy.invalid:8443",
+ "proxy auth failed for proxy-user with proxy-password\n",
+ ("proxy-user", "proxy-password"),
+ ),
+ (
+ "proxy-user:proxy-password@proxy.invalid:8443",
+ "proxy auth failed for proxy-user with proxy-password\n",
+ ("proxy-user", "proxy-password"),
+ ),
+ ],
+)
+def test_compose_redacts_host_proxy_credential_fragments(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ proxy_uri: str,
+ diagnostic: str,
+ secrets: tuple[str, ...],
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ split = max(1, len(diagnostic) // 2)
+ process = _StreamedComposeProcess([diagnostic[:split].encode(), diagnostic[split:].encode()])
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ monkeypatch.setenv("HTTPS_PROXY", proxy_uri)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def on_output(_text: str, _stream: str) -> None:
+ return None
+
+ result = asyncio.run(environment._run_docker_compose_command(["version"], check=False, on_output=on_output))
+
+ assert all(secret not in (result.stdout or "") for secret in secrets)
+
+
+@pytest.mark.parametrize(
+ ("environment_name", "credential_value"),
+ [
+ ("DOCKER_HOST", "tcp://compose-user:compose-password@docker.invalid:2376"),
+ ("HTTPS_PROXY", "https://proxy-user:proxy-password@proxy.invalid:8443"),
+ ("HTTPS_PROXY", "proxy-user:proxy-password@proxy.invalid:8443"),
+ (
+ "DOCKER_AUTH_CONFIG",
+ '{"auths":{"registry.invalid":{"auth":"compose-registry-credential"}}}',
+ ),
+ ],
+)
+def test_compose_redacts_host_client_credentials_from_checked_error(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ environment_name: str,
+ credential_value: str,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _BufferedComposeProcess(
+ stdout=f"client failure: {credential_value}\n".encode(),
+ return_code=7,
+ )
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ return process
+
+ monkeypatch.setenv(environment_name, credential_value)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(environment._run_docker_compose_command(["version"], check=True))
+
+ detail = str(caught.value)
+ assert credential_value not in detail
+ compose_credential_values = _compose_client_credential_values({environment_name: credential_value})
+ assert _marker_for(*compose_credential_values) in detail
+
+
+def test_compose_stream_callback_redacts_multiline_and_overlapping_secrets_across_reader_boundaries(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ multiline_secret = "FIRST-HALF\nSECOND-HALF"
+ shorter_secret = "overlap-secret"
+ longer_secret = "overlap-secret-tail"
+ nested_shorter_secret = "NESTED-SECRET\n"
+ nested_longer_secret = "NESTED-SECRET\nTAIL"
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([])
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ stdout = asyncio.StreamReader()
+ stdout.feed_data(b"prefix FIRST-")
+ stdout.feed_data(b"HALF\nSECOND-")
+ stdout.feed_data(b"HALF overlap-secret-tail overlap-")
+ stdout.feed_data(b"secret NESTED-SECRET\n")
+ stdout.feed_data(b"TAIL suffix\n")
+ stdout.feed_eof()
+ process.stdout = stdout
+ return await environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=on_output,
+ env_overrides={
+ "MULTILINE_SECRET": multiline_secret,
+ "SHORTER_SECRET": shorter_secret,
+ "LONGER_SECRET": longer_secret,
+ "NESTED_SHORTER_SECRET": nested_shorter_secret,
+ "NESTED_LONGER_SECRET": nested_longer_secret,
+ },
+ )
+
+ result = asyncio.run(exercise())
+ callback_output = "".join(callback_chunks)
+ marker = _marker_for(
+ multiline_secret,
+ shorter_secret,
+ longer_secret,
+ nested_shorter_secret,
+ nested_longer_secret,
+ )
+
+ assert callback_output == f"prefix {marker} {marker} {marker} {marker} suffix\n"
+ assert "FIRST-HALF" not in callback_output
+ assert "SECOND-HALF" not in callback_output
+ assert shorter_secret not in callback_output
+ assert longer_secret not in callback_output
+ assert "NESTED-SECRET" not in callback_output
+ assert "TAIL" not in callback_output
+ assert result.stdout == callback_output
+
+
+def test_compose_stream_callback_flushes_incomplete_secret_prefix_at_eof(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([b"ordinary output secret-"])
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ result = asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=on_output,
+ env_overrides={"SECRET": "secret-value"},
+ )
+ )
+
+ assert "".join(callback_chunks) == result.stdout == "ordinary output secret-"
+
+
+def test_streaming_and_buffered_redaction_union_is_deterministic_for_offset_overlaps() -> None:
+ script = """
+import json
+from skillevaluator.tier3.harbor.secure_docker_environment import _StreamingSecretRedactor, _redact
+
+secrets = {"abcdefgh", "ghijklmn"}
+text = "abcdefghijklmn\\n"
+redactor = _StreamingSecretRedactor(secrets)
+streamed = redactor.feed(text) + redactor.finish()
+print(json.dumps({"streamed": streamed, "buffered": _redact(text, secrets)}))
+"""
+ outputs: list[dict[str, str]] = []
+ for seed in ("1", "2", "3", "4", "5", "6", "7", "8"):
+ child_env = dict(os.environ)
+ child_env["PYTHONHASHSEED"] = seed
+ completed = subprocess.run(
+ [sys.executable, "-c", script],
+ cwd=Path(__file__).resolve().parents[1],
+ env=child_env,
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ outputs.append(json.loads(completed.stdout))
+
+ expected = _marker_for("abcdefgh", "ghijklmn") + "\n"
+ assert outputs == [{"streamed": expected, "buffered": expected}] * len(outputs)
+
+
+def test_compose_stream_callback_and_result_match_for_offset_overlap(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([b"abcdefghijklmn\n"])
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ result = asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=on_output,
+ env_overrides={"LEFT_SECRET": "abcdefgh", "RIGHT_SECRET": "ghijklmn"},
+ )
+ )
+
+ expected = _marker_for("abcdefgh", "ghijklmn") + "\n"
+ assert "".join(callback_chunks) == result.stdout == expected
+
+
+def test_compose_stream_callback_and_error_match_for_offset_overlap(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([b"abcdefghijklmn\n"], return_code=7)
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ on_output=on_output,
+ env_overrides={"LEFT_SECRET": "abcdefgh", "RIGHT_SECRET": "ghijklmn"},
+ )
+ )
+
+ expected = _marker_for("abcdefgh", "ghijklmn") + "\n"
+ assert "".join(callback_chunks) == expected
+ assert f"Stdout: {expected}." in str(caught.value)
+ assert f"abcdef{_marker_for('abcdefgh', 'ghijklmn')}" not in str(caught.value)
+
+
+@pytest.mark.parametrize(
+ ("secret_count", "output_size"),
+ [(100, 100_000), (500, 20_000)],
+)
+def test_streaming_redactor_scale_is_linear_in_input(
+ secret_count: int,
+ output_size: int,
+) -> None:
+ secrets = {f"secret-{index:04d}-value" for index in range(secret_count)}
+ selected_secret = f"secret-{secret_count - 1:04d}-value"
+ prefix_size = output_size // 2
+ text = "x" * prefix_size + selected_secret + "y" * (output_size - prefix_size)
+ redactor = _StreamingSecretRedactor(secrets, _track_transitions=True)
+
+ emitted: list[str] = []
+ longest_secret = max(map(len, secrets))
+ for index in range(0, len(text), 4093):
+ emitted.append(redactor.feed(text[index : index + 4093]))
+ assert len(redactor._pending) <= longest_secret - 1
+ emitted.append(redactor.finish())
+ streamed = "".join(emitted)
+
+ assert streamed == _redact(text, secrets)
+ assert selected_secret not in streamed
+ assert redactor.match_transition_count <= 2 * len(text)
+ assert redactor.match_work_count <= 10 * len(text)
+
+
+@pytest.mark.parametrize(
+ ("secret_length", "output_size"),
+ [(1024, 20_000), (4096, 5_000)],
+)
+def test_streaming_redactor_repeated_prefix_scan_is_linear(
+ secret_length: int,
+ output_size: int,
+) -> None:
+ secret = "a" * (secret_length - 1) + "b"
+ text = "a" * output_size
+ redactor = _StreamingSecretRedactor({secret}, _track_transitions=True)
+
+ assert redactor.feed(text) + redactor.finish() == text
+ assert redactor.match_transition_count <= 2 * len(text)
+ assert redactor.match_work_count <= 10 * len(text)
+
+
+def test_streaming_redactor_does_not_starve_event_loop_on_repeated_prefix() -> None:
+ async def exercise() -> float:
+ redactor = _StreamingSecretRedactor(
+ {"a" * 1023 + "b"},
+ _track_transitions=True,
+ )
+ text = "a" * 20_000
+ loop = asyncio.get_running_loop()
+ heartbeat_gaps: list[float] = []
+ keep_running = True
+
+ async def heartbeat() -> None:
+ previous = loop.time()
+ while keep_running:
+ await asyncio.sleep(0.01)
+ now = loop.time()
+ heartbeat_gaps.append(now - previous)
+ previous = now
+
+ heartbeat_task = asyncio.create_task(heartbeat())
+ await asyncio.sleep(0.02)
+ emitted = redactor.feed(text)
+ await asyncio.sleep(0.02)
+ keep_running = False
+ await heartbeat_task
+ assert emitted + redactor.finish() == text
+ return max(heartbeat_gaps)
+
+ assert asyncio.run(exercise()) < 1.0
+
+
+def test_streaming_redactor_nested_terminal_matching_work_is_linear() -> None:
+ secrets = {"a" * length for length in range(8, 1008)}
+ text = "a" * 20_000
+ started = time.perf_counter()
+ redactor = _StreamingSecretRedactor(secrets, _track_transitions=True)
+
+ output = redactor.feed(text) + redactor.finish()
+ elapsed = time.perf_counter() - started
+ assert elapsed < 1.0
+ assert output == _collision_safe_redaction_marker(secrets)
+ assert redactor.match_work_count <= 10 * len(text)
+
+
+def test_streaming_redactor_does_not_starve_event_loop_on_nested_terminals() -> None:
+ async def exercise() -> float:
+ redactor = _StreamingSecretRedactor(
+ {"a" * length for length in range(8, 1008)},
+ _track_transitions=True,
+ )
+ text = "a" * 20_000
+ loop = asyncio.get_running_loop()
+ heartbeat_gaps: list[float] = []
+ keep_running = True
+
+ async def heartbeat() -> None:
+ previous = loop.time()
+ while keep_running:
+ await asyncio.sleep(0.01)
+ now = loop.time()
+ heartbeat_gaps.append(now - previous)
+ previous = now
+
+ heartbeat_task = asyncio.create_task(heartbeat())
+ await asyncio.sleep(0.02)
+ emitted = redactor.feed(text)
+ await asyncio.sleep(0.02)
+ keep_running = False
+ await heartbeat_task
+ assert emitted + redactor.finish() == _collision_safe_redaction_marker(
+ {"a" * length for length in range(8, 1008)}
+ )
+ return max(heartbeat_gaps)
+
+ assert asyncio.run(exercise()) < 1.0
+
+
+def test_compose_stream_nested_secret_union_does_not_starve_event_loop(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([b"a" * 20_000])
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> tuple[ExecResult, float]:
+ loop = asyncio.get_running_loop()
+ heartbeat_gaps: list[float] = []
+ keep_running = True
+
+ async def heartbeat() -> None:
+ previous = loop.time()
+ while keep_running:
+ await asyncio.sleep(0.01)
+ now = loop.time()
+ heartbeat_gaps.append(now - previous)
+ previous = now
+
+ heartbeat_task = asyncio.create_task(heartbeat())
+ await asyncio.sleep(0.02)
+ result = await environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=on_output,
+ env_overrides={f"SECRET_{length}": "a" * length for length in range(8, 1008)},
+ )
+ await asyncio.sleep(0.02)
+ keep_running = False
+ await heartbeat_task
+ return result, max(heartbeat_gaps)
+
+ result, maximum_gap = asyncio.run(exercise())
+
+ expected = _collision_safe_redaction_marker({"a" * length for length in range(8, 1008)})
+ assert "".join(callback_chunks) == result.stdout == expected
+ assert maximum_gap < 1.0
+
+
+def _reference_union_redaction(text: str, secrets: set[str]) -> str:
+ eligible = {secret for secret in secrets if secret and len(secret) >= 8}
+ marker = _collision_safe_redaction_marker(eligible)
+ coverage = [0] * (len(text) + 1)
+ for secret in eligible:
+ match_start = text.find(secret)
+ while match_start >= 0:
+ coverage[match_start] += 1
+ coverage[match_start + len(secret)] -= 1
+ match_start = text.find(secret, match_start + 1)
+
+ redacted: list[str] = []
+ active_matches = 0
+ redaction_open = False
+ for position, character in enumerate(text):
+ active_matches += coverage[position]
+ if active_matches:
+ if not redaction_open:
+ redacted.append(marker)
+ redaction_open = True
+ else:
+ redaction_open = False
+ redacted.append(character)
+ return "".join(redacted)
+
+
+@pytest.mark.parametrize(
+ ("secrets", "text", "expected"),
+ [
+ ({"abcdefgh", "abcdefghij"}, "abcdefghij", "[REDACTED]"),
+ ({"abcdefgh", "bcdefghijk"}, "abcdefghijk", "[REDACTED]"),
+ ({"abcdefgh", "ghijklmn"}, "abcdefghijklmn", "[REDACTED]"),
+ ({"aaaaaaaa", "aaaaaaaaaa"}, "a" * 12, "[REDACTED]"),
+ ({"xxabcdefgh", "abcdefgh"}, "xxabcdefgh", "[REDACTED]"),
+ ({"abcdefgh", "ijklmnop"}, "abcdefghijklmnop", "[REDACTED]"),
+ ({"abcdefgh", "jklmnopq"}, "abcdefghXjklmnopq", "[REDACTED]X[REDACTED]"),
+ (
+ {"abcdefgh", "klmnopqr", "ghijklmnopqrst"},
+ "abcdefghijklmnopqrst",
+ "[REDACTED]",
+ ),
+ (
+ {"unicode-🔑alpha\nβ", "🔑alpha\nβ"},
+ "prefix unicode-🔑alpha\nβ suffix",
+ "prefix [REDACTED] suffix",
+ ),
+ ],
+)
+def test_streaming_redactor_matches_union_reference_for_every_single_split(
+ secrets: set[str],
+ text: str,
+ expected: str,
+) -> None:
+ marker = _collision_safe_redaction_marker(secrets)
+ expected = expected.replace("[REDACTED]", marker)
+ assert _reference_union_redaction(text, secrets) == expected
+ longest_secret = max(map(len, secrets))
+
+ for split in range(len(text) + 1):
+ redactor = _StreamingSecretRedactor(secrets)
+ streamed = redactor.feed(text[:split])
+ assert len(redactor._pending) <= longest_secret - 1
+ streamed += redactor.feed(text[split:])
+ assert len(redactor._pending) <= longest_secret - 1
+ streamed += redactor.finish()
+ assert streamed == expected
+ assert _redact(text, secrets) == expected
+
+
+def test_streaming_redactor_matches_union_reference_for_randomized_chunks() -> None:
+ randomizer = random.Random(0xAC022)
+ alphabet = "abXYé🙂\n"
+
+ for _case in range(100):
+ secrets = {
+ "".join(randomizer.choice(alphabet) for _character in range(randomizer.randint(8, 24)))
+ for _secret in range(randomizer.randint(1, 20))
+ }
+ # Include nested and shared-prefix patterns on every run so overlapping
+ # coverage and automaton failure behavior are exercised independently of chance.
+ secrets.update({"aaaaaaaa", "aaaaaaaaaa", "abcdefgh", "abcdefghij"})
+ parts = [
+ "".join(randomizer.choice(alphabet) for _character in range(randomizer.randint(0, 30)))
+ for _part in range(randomizer.randint(2, 8))
+ ]
+ selected = randomizer.sample(sorted(secrets), k=min(len(parts) - 1, len(secrets)))
+ text = "".join(part + (selected[index] if index < len(selected) else "") for index, part in enumerate(parts))
+ expected = _reference_union_redaction(text, secrets)
+ longest_secret = max(map(len, secrets))
+ redactor = _StreamingSecretRedactor(secrets, _track_transitions=True)
+ emitted: list[str] = []
+ position = 0
+ while position < len(text):
+ chunk_size = randomizer.randint(1, 19)
+ emitted.append(redactor.feed(text[position : position + chunk_size]))
+ position += chunk_size
+ assert len(redactor._pending) <= longest_secret - 1
+ emitted.append(redactor.finish())
+
+ streamed = "".join(emitted)
+ assert streamed == expected
+ assert _redact(text, secrets) == expected
+ for secret in secrets:
+ assert secret not in streamed
+ assert redactor.match_transition_count <= 2 * len(text)
+ assert redactor.match_work_count <= 10 * len(text)
+
+
+def test_collision_safe_marker_invariant_with_unicode_and_occupied_candidates() -> None:
+ occupied_candidates = "".join(_REDACTION_SENTINEL_CANDIDATES)
+ secrets = {
+ f"{occupied_candidates}unicode-🔑alpha\nβ",
+ "abcdefgh",
+ "x[REDACTED]",
+ }
+
+ marker = _collision_safe_redaction_marker(secrets)
+ sentinel = marker[0]
+ minimum_secret_length = min(map(len, secrets))
+
+ assert marker[-1] == sentinel
+ assert all(sentinel not in secret for secret in secrets)
+ assert all(secret not in marker for secret in secrets)
+ for start in range(len(marker) - minimum_secret_length + 1):
+ assert sentinel in marker[start : start + minimum_secret_length]
+ for secret in secrets:
+ for split in range(1, len(secret)):
+ assert secret not in f"{secret[:split]}{marker}{secret[split:]}"
+
+
+def test_collision_safe_marker_falls_back_to_an_absent_unicode_scalar(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "_REDACTION_SENTINEL_CANDIDATES",
+ (),
+ )
+ secret = "abcdefgh\ue000"
+
+ marker = secure_docker_environment._collision_safe_redaction_marker({secret})
+
+ assert marker[0] == "\ue001"
+ assert marker[-1] == "\ue001"
+ assert secret not in marker
+
+
+def test_collision_safe_marker_fallback_never_emits_terminal_controls(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "_REDACTION_SENTINEL_CANDIDATES",
+ (),
+ )
+ occupied_private_use_and_controls = "".join(
+ chr(codepoint)
+ for candidate_range in (
+ range(0xE000, 0xF900),
+ range(0xF0000, 0xFFFFE),
+ range(0x100000, 0x10FFFE),
+ range(1, 0x1B),
+ )
+ for codepoint in candidate_range
+ )
+
+ marker = secure_docker_environment._collision_safe_redaction_marker({occupied_private_use_and_controls})
+
+ assert marker[0].isprintable()
+ assert not marker[0].isspace()
+ assert marker[-1] == marker[0]
+
+
+def test_collision_safe_marker_exhaustion_fails_before_process_creation(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "_REDACTION_SENTINEL_CANDIDATES",
+ (),
+ )
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "unicodedata",
+ SimpleNamespace(category=lambda _candidate: "Cc"),
+ )
+ occupied_private_use = "".join(
+ chr(codepoint)
+ for candidate_range in (
+ range(0xE000, 0xF900),
+ range(0xF0000, 0xFFFFE),
+ range(0x100000, 0x10FFFE),
+ )
+ for codepoint in candidate_range
+ )
+ environment = _initialized_docker_environment(tmp_path)
+ subprocess_created = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ nonlocal subprocess_created
+ subprocess_created = True
+ return _BufferedComposeProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="Could not construct a collision-safe redaction marker") as caught:
+ asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "main", "true"],
+ env_overrides={"SECRET": occupied_private_use},
+ )
+ )
+
+ assert not subprocess_created
+ assert occupied_private_use not in str(caught.value)
+
+
+@pytest.mark.parametrize("check", [True, False])
+def test_compose_stream_callback_nonzero_check_semantics_are_redacted(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ check: bool,
+) -> None:
+ secret = "nonzero-secret-value"
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([f"failure {secret}\n".encode()], return_code=7)
+ callbacks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callbacks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ run = environment._run_docker_compose_command(
+ ["exec", "main", "fail"],
+ check=check,
+ on_output=on_output,
+ env_overrides={"SECRET": secret},
+ )
+
+ if check:
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(run)
+ assert secret not in str(caught.value)
+ assert _marker_for(secret) in str(caught.value)
+ else:
+ result = asyncio.run(run)
+ assert result.return_code == 7
+ assert result.stdout == f"failure {_marker_for(secret)}\n"
+ assert "".join(callbacks) == f"failure {_marker_for(secret)}\n"
+
+
+def test_compose_stream_check_failure_redacts_replacement_token_once(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ secret = "REDACTED"
+ environment = _initialized_docker_environment(
+ tmp_path,
+ environment_name=f"env-{secret}",
+ )
+ process = _StreamedComposeProcess([f"failure {secret}\n".encode()], return_code=7)
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "main", "fail", secret],
+ on_output=on_output,
+ env_overrides={"SECRET": secret},
+ )
+ )
+
+ detail = str(caught.value)
+ marker = _marker_for(secret)
+ assert "".join(callback_chunks) == f"failure {marker}\n"
+ assert secret not in "".join(callback_chunks)
+ assert secret not in detail
+ assert f"environment env-{marker}" in detail
+ assert f"fail {marker}" in detail
+ assert f"Stdout: failure {marker}\n." in detail
+ assert "env-REDACTED" not in detail
+ assert "fail REDACTED" not in detail
+ assert "failure REDACTED" not in detail
+ assert detail.count(marker) == 3
+
+
+@pytest.mark.parametrize(
+ ("secrets", "raw_output"),
+ [
+ (("REDACTED", "[REDACTED]"), "REDACTED [REDACTED]\n"),
+ (("12345678", "x[REDACTED]"), "x12345678\n"),
+ ],
+)
+def test_compose_stream_redaction_marker_cannot_disclose_or_synthesize_secrets(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ secrets: tuple[str, ...],
+ raw_output: str,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ callbacks: list[list[str]] = [[], []]
+ callback_index = 0
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return _StreamedComposeProcess([raw_output.encode()], return_code=7)
+
+ async def on_output(text: str, _stream: str) -> None:
+ callbacks[callback_index].append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ environment_overrides = {f"SECRET_{index}": secret for index, secret in enumerate(secrets)}
+
+ async def exercise() -> tuple[ExecResult, RuntimeError]:
+ nonlocal callback_index
+ result = await environment._run_docker_compose_command(
+ ["exec", "main", "fail"],
+ check=False,
+ on_output=on_output,
+ env_overrides=environment_overrides,
+ )
+ callback_index = 1
+ with pytest.raises(RuntimeError) as caught:
+ await environment._run_docker_compose_command(
+ ["exec", "main", "fail"],
+ on_output=on_output,
+ env_overrides=environment_overrides,
+ )
+ return result, caught.value
+
+ result, error = asyncio.run(exercise())
+ check_false_callback = "".join(callbacks[0])
+ check_true_callback = "".join(callbacks[1])
+
+ assert check_false_callback == result.stdout
+ assert check_true_callback == result.stdout
+ for rendered in (check_false_callback, result.stdout or "", str(error)):
+ for secret in secrets:
+ assert secret not in rendered
+
+
+@pytest.mark.parametrize(
+ "error_type",
+ [TimeoutError, LookupError, _CallbackBaseError, asyncio.CancelledError],
+)
+def test_compose_stream_callback_exception_is_propagated_after_process_reap(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ error_type: type[BaseException],
+) -> None:
+ secret = "callback-secret-value"
+ monkeypatch.setenv("SSH_AUTH_SOCK", "/tmp/" + "noncredential-socket-path-" * 4)
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([f"output {secret}\n".encode(), b"unread tail\n"])
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def failing_callback(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+ raise error_type("stream consumer failed")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ with pytest.raises(error_type, match="stream consumer failed") as caught:
+ asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=failing_callback,
+ env_overrides={"SECRET": secret},
+ )
+ )
+
+ assert callback_chunks == [f"output {_marker_for(secret)}"]
+ assert secret not in str(caught.value)
+ assert process.returncode is not None
+ assert process.wait_count >= 1
+
+
+def test_compose_stream_external_cancellation_reaps_cooperative_callback_task(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([b"callback starts\n"])
+ callback_started = asyncio.Event()
+ callback_reaped = asyncio.Event()
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_CANCEL_SECONDS", 0.01)
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def cooperative_callback(_text: str, _stream: str) -> None:
+ callback_started.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ callback_reaped.set()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> None:
+ task = asyncio.create_task(
+ environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=cooperative_callback,
+ )
+ )
+ await asyncio.wait_for(callback_started.wait(), timeout=1)
+ task.cancel()
+ await asyncio.sleep(0)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=1)
+ await asyncio.wait_for(callback_reaped.wait(), timeout=1)
+ await asyncio.sleep(0)
+ current = asyncio.current_task()
+ assert all(candidate is current or candidate.done() for candidate in asyncio.all_tasks())
+
+ asyncio.run(exercise())
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
+@pytest.mark.parametrize(
+ "error_type",
+ [TimeoutError, _CallbackBaseError, asyncio.CancelledError],
+)
+def test_real_subprocess_callback_failure_repeat_preserves_primary_exception(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ error_type: type[BaseException],
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ real_create_subprocess = asyncio.create_subprocess_exec
+ processes: list[asyncio.subprocess.Process] = []
+ callback_chunks: list[str] = []
+ secret = "real-subprocess-callback-secret"
+
+ async def create_real_subprocess(*_args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await real_create_subprocess(
+ sys.executable,
+ "-c",
+ "import sys, time; print(sys.argv[1], flush=True); time.sleep(60)",
+ secret,
+ stdin=kwargs["stdin"],
+ stdout=kwargs["stdout"],
+ stderr=kwargs["stderr"],
+ start_new_session=kwargs["start_new_session"],
+ )
+ processes.append(process)
+ return process
+
+ async def failing_callback(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+ raise error_type("real callback failure")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_real_subprocess)
+
+ async def exercise() -> None:
+ try:
+ for _index in range(10):
+ with pytest.raises(error_type, match="real callback failure") as caught:
+ await environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=failing_callback,
+ env_overrides={"SECRET": secret},
+ )
+ assert secret not in str(caught.value)
+ assert processes[-1].returncode is not None
+ finally:
+ for process in processes:
+ if process.returncode is None:
+ with contextlib.suppress(ProcessLookupError):
+ process.kill()
+ await process.wait()
+
+ asyncio.run(exercise())
+
+ assert len(processes) == 10
+ assert callback_chunks == [_marker_for(secret)] * 10
+
+
+@pytest.mark.skipif(os.name != "posix", reason="requires POSIX process groups")
+def test_signal_process_tree_suppresses_permission_race_only_after_leader_exit(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ process = SimpleNamespace(pid=99881, returncode=None)
+ monkeypatch.setattr(os, "killpg", lambda *_args: (_ for _ in ()).throw(PermissionError("denied")))
+
+ def missing_process(_pid: int) -> int:
+ raise ProcessLookupError
+
+ monkeypatch.setattr(os, "getpgid", missing_process)
+ _signal_process_tree(process, signal.SIGTERM) # type: ignore[arg-type]
+
+ monkeypatch.setattr(os, "getpgid", lambda pid: pid)
+ with pytest.raises(PermissionError, match="denied"):
+ _signal_process_tree(process, signal.SIGTERM) # type: ignore[arg-type]
+
+
+def test_callback_primary_exception_retains_cleanup_failure_evidence(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _StreamedComposeProcess([b"callback output\n"])
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _StreamedComposeProcess:
+ return process
+
+ async def failing_callback(_text: str, _stream: str) -> None:
+ raise _CallbackBaseError("primary callback failure")
+
+ async def cleanup_then_fail(
+ _process: object,
+ communication: asyncio.Task[object],
+ *,
+ contain_service_on_interrupt: str | None = None,
+ stop_main_on_interrupt: bool,
+ ) -> None:
+ assert contain_service_on_interrupt is None
+ del stop_main_on_interrupt
+ await communication
+ raise PermissionError("cleanup denied")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(environment, "_contain_main_and_reap_compose", cleanup_then_fail)
+
+ with pytest.raises(_CallbackBaseError, match="primary callback failure") as caught:
+ asyncio.run(
+ environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=failing_callback,
+ )
+ )
+
+ assert isinstance(caught.value.__cause__, PermissionError)
+ assert "cleanup denied" in str(caught.value.__cause__)
+ assert any("cleanup or container containment/restoration also failed" in note for note in caught.value.__notes__)
+
+
def _write_skill(tmp_path: Path) -> Path:
skill = tmp_path / "skill"
evals = skill / "evals"
@@ -36,51 +1564,5300 @@ def _write_skill(tmp_path: Path) -> Path:
json.dumps([{"id": "case-001", "question": "Do it", "expected_skill": "skill"}]),
encoding="utf-8",
)
- return skill
+ return skill
+
+
+def test_generated_tasks_stage_only_names_and_placeholders(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("NVIDIA_API_KEY", _SENTINEL)
+ task = generate_harbor_tasks(
+ _write_skill(tmp_path),
+ tmp_path / "tasks",
+ runtime_env={"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"},
+ verifier_env={
+ "NVIDIA_API_KEY": "${NVIDIA_API_KEY}",
+ "OPENAI_API_KEY": "${OPENAI_API_KEY}",
+ },
+ )[0]
+
+ staged_text = "\n".join(
+ path.read_text(encoding="utf-8", errors="replace") for path in task.rglob("*") if path.is_file()
+ )
+ assert _SENTINEL not in staged_text
+ assert 'NVIDIA_API_KEY = "${NVIDIA_API_KEY}"' in staged_text
+ assert 'OPENAI_API_KEY = "${OPENAI_API_KEY}"' in staged_text
+
+
+def test_docker_command_uses_secure_environment_import_path() -> None:
+ command = build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="opencode",
+ job_name="secure-docker",
+ env_mode="docker",
+ )
+
+ assert "--agent-import-path" not in command
+ assert "--environment-import-path" not in command
+ assert command[command.index("--agent") + 1] == "opencode"
+ assert command[command.index("--env") + 1] == SECURE_DOCKER_ENV_IMPORT_PATH
+
+
+@pytest.mark.parametrize("secret", ["x", "hunter2", "public-exec-callback-secret"])
+def test_public_exec_streams_through_harbor_scoped_output_callback(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ secret: str,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ process = _BufferedAndStreamedComposeProcess(
+ [f"output {secret}\n".encode(), b"tail\n"],
+ )
+ callback_chunks: list[tuple[str, str]] = []
+ captured: dict[str, object] = {}
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedAndStreamedComposeProcess:
+ captured["args"] = args
+ captured["env"] = kwargs["env"]
+ return process
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ "emit-output",
+ env={"PUBLIC_EXEC_TOKEN": secret},
+ )
+
+ result = asyncio.run(exercise())
+ marker = _collision_safe_redaction_marker({secret}, include_short=True)
+ expected = f"output {marker}\ntail\n"
+
+ assert "".join(text for text, _stream in callback_chunks) == result.stdout == expected
+ assert {stream for _text, stream in callback_chunks} == {"stdout"}
+ rendered_arguments = [str(argument) for argument in captured["args"]]
+ if len(secret) >= 8:
+ assert all(secret not in argument for argument in rendered_arguments)
+ else:
+ assert secret not in rendered_arguments
+ assert isinstance(captured["env"], dict)
+ assert captured["env"]["PUBLIC_EXEC_TOKEN"] == secret
+
+
+@pytest.mark.parametrize("service", [None, MAIN_SERVICE_NAME])
+def test_service_exec_for_main_delegates_to_secure_public_exec(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ service: str | None,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ captured: dict[str, object] = {}
+ expected = ExecResult(stdout="secure-main\n", stderr=None, return_code=0)
+
+ async def secure_exec(
+ command: str,
+ cwd: str | None = None,
+ env: dict[str, str] | None = None,
+ timeout_sec: int | None = None,
+ user: str | int | None = None,
+ ) -> ExecResult:
+ captured.update(
+ command=command,
+ cwd=cwd,
+ env=env,
+ timeout_sec=timeout_sec,
+ user=user,
+ )
+ return expected
+
+ monkeypatch.setattr(environment, "exec", secure_exec)
+
+ result = asyncio.run(
+ environment.service_exec(
+ "main-command",
+ service=service,
+ cwd="/main-cwd",
+ env={"MAIN_TOKEN": "main-explicit-secret"},
+ timeout_sec=17,
+ user=1200,
+ )
+ )
+
+ assert result == expected
+ assert captured == {
+ "command": "main-command",
+ "cwd": "/main-cwd",
+ "env": {"MAIN_TOKEN": "main-explicit-secret"},
+ "timeout_sec": 17,
+ "user": 1200,
+ }
+
+
+def test_sidecar_service_exec_keeps_values_off_argv_and_isolates_main_environment(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persistent_secret = "main-persistent-secret-75124"
+ task_secret = "main-task-secret-86235"
+ scoped_secret = "main-scoped-secret-97346"
+ sidecar_secret = "sidecar-explicit-secret-08457"
+ reused_name_secret = "sidecar-reused-name-secret-19568"
+ main_secrets = {persistent_secret, task_secret, scoped_secret}
+ sidecar_secrets = {sidecar_secret, reused_name_secret}
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"PERSISTENT_TOKEN": persistent_secret},
+ )
+ environment.default_user = 4321
+ environment.task_env_config = EnvironmentConfig(workdir="/main-only-workdir")
+ environment._compose_task_env = {
+ "TASK_TOKEN": task_secret,
+ "TASK_WRAPPED": f"prefix:{persistent_secret}:suffix",
+ }
+ monkeypatch.setenv("SCOPED_TOKEN", scoped_secret)
+ monkeypatch.setenv("HOST_WRAPPED_MAIN_TOKEN", f"prefix:{task_secret}:suffix")
+ monkeypatch.setenv("NVIDIA_API_KEY", NVIDIA_BUILD_STDIN_SENTINEL)
+ monkeypatch.setenv(NVIDIA_BUILD_KEY_STDIN_ENV, "1")
+ monkeypatch.setenv("SKILLEVALUATOR_NVIDIA_API_KEY_FILE", "/tmp/main-only-nvidia-key")
+
+ process = _BufferedAndStreamedComposeProcess(
+ [
+ f"stdout {sidecar_secret}\n".encode(),
+ f"stderr {reused_name_secret}\n".encode(),
+ ],
+ return_code=9,
+ )
+ captured: dict[str, object] = {}
+ callback_chunks: list[tuple[str, str]] = []
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedAndStreamedComposeProcess:
+ captured["args"] = args
+ captured["env"] = dict(kwargs["env"])
+ return process
+
+ async def on_output(text: str, stream: str) -> None:
+ callback_chunks.append((text, stream))
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with (
+ environment.scoped_exec_env({"SCOPED_TOKEN": scoped_secret}),
+ environment.scoped_output_callback(on_output),
+ ):
+ return await environment.service_exec(
+ "printf sidecar-output; printf sidecar-error >&2; exit 9",
+ service="helper",
+ env={
+ "SIDECAR_TOKEN": sidecar_secret,
+ # Reusing a main-only name is intentional and must retain
+ # the explicit sidecar value after base-env filtering.
+ "PERSISTENT_TOKEN": reused_name_secret,
+ },
+ )
+
+ result = asyncio.run(exercise())
+ marker = _collision_safe_redaction_marker(sidecar_secrets)
+ expected = f"stdout {marker}\nstderr {marker}\n"
+ arguments = [str(argument) for argument in captured["args"]] # type: ignore[index]
+ process_environment = captured["env"]
+
+ assert isinstance(process_environment, dict)
+ exec_arguments = arguments[arguments.index("exec") :]
+ carrier_names = {name for name in process_environment if name.startswith("SKILLEVALUATOR_SIDECAR_ENV_")}
+ assert len(carrier_names) == 2
+ assert exec_arguments[0] == "exec"
+ assert exec_arguments[1:5:2] == ["-e", "-e"]
+ assert set(exec_arguments[2:6:2]) == carrier_names
+ assert exec_arguments[-6:-3] == ["helper", "sh", "-c"]
+ assert exec_arguments[-2:] == ["sh", "printf sidecar-output; printf sidecar-error >&2; exit 9"]
+ wrapper = exec_arguments[-3]
+ assert all(f"export {name}=" in wrapper for name in ("SIDECAR_TOKEN", "PERSISTENT_TOKEN"))
+ assert all(carrier in wrapper for carrier in carrier_names)
+ assert 'exec /bin/sh -c "$1"' in wrapper
+ assert "bash" not in exec_arguments[-6:]
+ assert "-w" not in arguments
+ assert "-u" not in arguments
+ assert all(secret not in " ".join(arguments) for secret in main_secrets | sidecar_secrets)
+ assert "SIDECAR_TOKEN" not in process_environment
+ assert "PERSISTENT_TOKEN" not in process_environment
+ assert {process_environment[name] for name in carrier_names} == sidecar_secrets
+ assert (
+ not {
+ "TASK_TOKEN",
+ "TASK_WRAPPED",
+ "SCOPED_TOKEN",
+ "HOST_WRAPPED_MAIN_TOKEN",
+ "NVIDIA_API_KEY",
+ NVIDIA_BUILD_KEY_STDIN_ENV,
+ "SKILLEVALUATOR_NVIDIA_API_KEY_FILE",
+ }
+ & process_environment.keys()
+ )
+ assert all(
+ secret not in value
+ for value in process_environment.values()
+ if isinstance(value, str)
+ for secret in main_secrets
+ )
+ assert "".join(text for text, _stream in callback_chunks) == result.stdout == expected
+ assert {stream for _text, stream in callback_chunks} == {"stdout"}
+ assert result.return_code == 9
+
+
+@pytest.mark.parametrize("secret", ["x", "hunter2", "abcdefgh"])
+def test_sidecar_sensitive_named_values_redact_exact_short_and_long_output(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ secret: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ process = _BufferedAndStreamedComposeProcess(
+ [f"before|{secret}|after\n".encode()],
+ return_code=7,
+ )
+ callback_chunks: list[str] = []
+ captured_arguments: tuple[object, ...] = ()
+
+ async def create_subprocess(
+ *args: object,
+ **_kwargs: object,
+ ) -> _BufferedAndStreamedComposeProcess:
+ nonlocal captured_arguments
+ captured_arguments = args
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(on_output):
+ return await environment.service_exec(
+ "emit-sensitive-output",
+ service="helper",
+ env={"API_TOKEN": secret},
+ )
+
+ result = asyncio.run(exercise())
+ marker = _collision_safe_redaction_marker({secret}, include_short=True)
+
+ assert "".join(callback_chunks) == result.stdout == f"before|{marker}|after\n"
+ assert secret not in (result.stdout or "")
+ assert secret not in captured_arguments
+ assert result.return_code == 7
+
+
+def test_sidecar_exec_redacts_schemeless_proxy_components(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ process = _BufferedAndStreamedComposeProcess(
+ [b"proxy rejected sidecar-user with sidecar-password\n"],
+ return_code=7,
+ )
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedAndStreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(on_output):
+ return await environment.service_exec(
+ "emit-proxy-failure",
+ service="helper",
+ env={"HTTPS_PROXY": "sidecar-user:sidecar-password@proxy.invalid:8443"},
+ )
+
+ result = asyncio.run(exercise())
+ rendered = "".join(callback_chunks) + (result.stdout or "") + (result.stderr or "")
+
+ assert result.return_code == 7
+ assert "sidecar-user" not in rendered
+ assert "sidecar-password" not in rendered
+
+
+def test_sidecar_main_only_redaction_values_include_proxy_components(tmp_path: Path) -> None:
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={
+ "HTTPS_PROXY": "main-user:main-password@proxy.invalid:8443",
+ },
+ )
+
+ names, values = environment._main_only_compose_environment()
+
+ assert "HTTPS_PROXY" in names
+ assert {"main-user", "main-password"} <= values
+
+
+def test_sidecar_service_exec_uses_only_explicit_workdir_and_user(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment.default_user = 4321
+ environment.task_env_config = EnvironmentConfig(workdir="/main-only-workdir")
+ captured: dict[str, object] = {}
+
+ async def capture(
+ command: list[str],
+ check: bool = True,
+ timeout_sec: float | None = None,
+ stdin_data: bytes | None = None,
+ on_output: object | None = None,
+ **kwargs: object,
+ ) -> ExecResult:
+ captured.update(
+ command=command,
+ check=check,
+ timeout_sec=timeout_sec,
+ stdin_data=stdin_data,
+ on_output=on_output,
+ kwargs=kwargs,
+ )
+ return ExecResult(stdout="ok", stderr=None, return_code=0)
+
+ monkeypatch.setattr(environment, "_run_docker_compose_command", capture)
+
+ result = asyncio.run(
+ environment.service_exec(
+ "pwd",
+ service="helper",
+ cwd="/sidecar-workdir",
+ timeout_sec=23,
+ user=2222,
+ )
+ )
+
+ assert result.return_code == 0
+ assert captured["command"] == [
+ "exec",
+ "-w",
+ "/sidecar-workdir",
+ "-u",
+ "2222",
+ "--",
+ "helper",
+ "sh",
+ "-c",
+ "pwd",
+ ]
+ assert captured["check"] is False
+ assert captured["timeout_sec"] == 23
+ assert captured["stdin_data"] is None
+ assert captured["kwargs"] == {
+ "sidecar_env_carriers": {},
+ "additional_secret_values": {
+ NVIDIA_BUILD_STDIN_SENTINEL,
+ "skillevaluator-file-backed-nvidia-key",
+ },
+ "compose_env_excluded_names": {
+ "NVIDIA_API_KEY",
+ NVIDIA_BUILD_KEY_STDIN_ENV,
+ "SKILLEVALUATOR_NVIDIA_API_KEY_FILE",
+ },
+ "compose_env_excluded_values": {
+ NVIDIA_BUILD_STDIN_SENTINEL,
+ "skillevaluator-file-backed-nvidia-key",
+ },
+ "use_sidecar_compose_model": True,
+ "contain_service_on_interrupt": "helper",
+ "stop_main_on_interrupt": False,
+ }
+
+
+def test_sidecar_filter_removes_all_shadowed_main_values_before_explicit_reuse(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persistent_value = "shadowed-persistent-main-value"
+ task_value = "shadowed-task-main-value"
+ scoped_value = "shadowed-scoped-main-value"
+ sidecar_value = "intentional-sidecar-shared-value"
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"SHARED_TOKEN": persistent_value},
+ )
+ environment._compose_task_env = {"SHARED_TOKEN": task_value}
+ monkeypatch.setenv("WRAPPED_PERSISTENT", f"prefix:{persistent_value}:suffix")
+ monkeypatch.setenv("WRAPPED_TASK", f"prefix:{task_value}:suffix")
+ monkeypatch.setenv("WRAPPED_SCOPED", f"prefix:{scoped_value}:suffix")
+ captured_environment: dict[str, str] = {}
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ captured_environment.update(kwargs["env"])
+ return _BufferedComposeProcess(stdout=b"")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_exec_env({"SHARED_TOKEN": scoped_value}):
+ return await environment.service_exec(
+ "true",
+ service="helper",
+ env={"SHARED_TOKEN": sidecar_value},
+ )
+
+ result = asyncio.run(exercise())
+
+ assert result.return_code == 0
+ assert "SHARED_TOKEN" not in captured_environment
+ carriers = {
+ name: value for name, value in captured_environment.items() if name.startswith("SKILLEVALUATOR_SIDECAR_ENV_")
+ }
+ assert set(carriers.values()) == {sidecar_value}
+ assert not {"WRAPPED_PERSISTENT", "WRAPPED_TASK", "WRAPPED_SCOPED"} & captured_environment.keys()
+ assert all(
+ main_value not in value
+ for value in captured_environment.values()
+ for main_value in (persistent_value, task_value, scoped_value)
+ )
+
+
+def test_sidecar_service_name_is_validated_and_option_terminated(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ commands: list[list[str]] = []
+
+ async def capture(command: list[str], **_kwargs: object) -> ExecResult:
+ commands.append(command)
+ return ExecResult(stdout="ok", stderr=None, return_code=0)
+
+ monkeypatch.setattr(environment, "_run_docker_compose_command", capture)
+
+ result = asyncio.run(environment.service_exec("true", service="-T"))
+
+ assert result.return_code == 0
+ assert commands == [["exec", "--", "-T", "sh", "-c", "true"]]
+
+ for invalid_service in ("", "helper/name", "helper name", "helper\x00name"):
+ with pytest.raises(ValueError, match="Invalid Docker Compose service name"):
+ asyncio.run(environment.service_exec("true", service=invalid_service))
+ assert len(commands) == 1
+
+
+def test_sidecar_invalid_target_environment_fails_before_spawn_without_rendering_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ secret = "invalid-target-name-secret-value"
+ spawned = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ nonlocal spawned
+ spawned = True
+ return _BufferedComposeProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(ValueError, match="Invalid environment variable name") as caught:
+ asyncio.run(
+ environment.service_exec(
+ "true",
+ service="helper",
+ env={"INVALID-NAME": secret},
+ )
+ )
+
+ assert secret not in str(caught.value)
+ assert spawned is False
+
+
+def test_same_sidecar_execs_are_serialized_before_target_containment(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+
+ async def exercise() -> bool:
+ first_started = asyncio.Event()
+ release_first = asyncio.Event()
+ second_started = asyncio.Event()
+ call_count = 0
+
+ async def controlled_run(_command: list[str], **_kwargs: object) -> ExecResult:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ first_started.set()
+ await release_first.wait()
+ return ExecResult(stdout="first", stderr=None, return_code=0)
+ second_started.set()
+ return ExecResult(stdout="second", stderr=None, return_code=0)
+
+ monkeypatch.setattr(environment, "_run_docker_compose_command", controlled_run)
+ first = asyncio.create_task(environment.service_exec("first", service="helper"))
+ await asyncio.wait_for(first_started.wait(), timeout=1)
+ second = asyncio.create_task(environment.service_exec("second", service="helper"))
+ await asyncio.sleep(0)
+ overlapped = second_started.is_set()
+ release_first.set()
+ assert (await first).stdout == "first"
+ assert (await second).stdout == "second"
+ return overlapped
+
+ assert asyncio.run(exercise()) is False
+
+
+def test_cross_sidecar_callback_lock_cycle_fails_fast_without_deadlock(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+
+ async def exercise() -> list[tuple[str, str]]:
+ both_outer_locks = asyncio.Event()
+ entered = 0
+ outcomes: list[tuple[str, str]] = []
+
+ async def nested(outer: str, inner: str) -> None:
+ nonlocal entered
+ async with environment._sidecar_operation(outer):
+ entered += 1
+ if entered == 2:
+ both_outer_locks.set()
+ await both_outer_locks.wait()
+ try:
+ async with environment._sidecar_operation(inner):
+ outcomes.append((outer, "entered"))
+ except RuntimeError:
+ outcomes.append((outer, "rejected"))
+
+ await asyncio.wait_for(
+ asyncio.gather(
+ nested("helper-a", "helper-b"),
+ nested("helper-b", "helper-a"),
+ ),
+ timeout=1,
+ )
+ return outcomes
+
+ assert sorted(asyncio.run(exercise())) == [
+ ("helper-a", "entered"),
+ ("helper-b", "rejected"),
+ ]
+
+
+def test_cross_environment_sidecar_lock_cycle_fails_fast_without_deadlock(
+ tmp_path: Path,
+) -> None:
+ (tmp_path / "one").mkdir()
+ (tmp_path / "two").mkdir()
+ environments = [
+ _initialized_secure_docker_environment(tmp_path / "one"),
+ _initialized_secure_docker_environment(tmp_path / "two"),
+ ]
+ lower, higher = sorted(environments, key=id)
+
+ async def exercise() -> list[str]:
+ both_outer_locks = asyncio.Event()
+ entered = 0
+ outcomes: list[str] = []
+
+ async def nested(
+ outer_environment: SkillEvaluatorSecureDockerEnvironment,
+ outer_service: str,
+ inner_environment: SkillEvaluatorSecureDockerEnvironment,
+ inner_service: str,
+ label: str,
+ ) -> None:
+ nonlocal entered
+ async with outer_environment._sidecar_operation(outer_service):
+ entered += 1
+ if entered == 2:
+ both_outer_locks.set()
+ await both_outer_locks.wait()
+ try:
+ async with inner_environment._sidecar_operation(inner_service):
+ outcomes.append(f"{label}:entered")
+ except RuntimeError:
+ outcomes.append(f"{label}:rejected")
+
+ await asyncio.wait_for(
+ asyncio.gather(
+ nested(lower, "helper-a", higher, "helper-b", "ascending"),
+ nested(higher, "helper-b", lower, "helper-a", "descending"),
+ ),
+ timeout=1,
+ )
+ return outcomes
+
+ assert sorted(asyncio.run(exercise())) == [
+ "ascending:entered",
+ "descending:rejected",
+ ]
+
+
+def test_raw_service_resolution_uses_stdout_ids_and_ignores_stderr_warning(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ identity = "1" * 64
+ captured_commands: list[list[str]] = []
+
+ async def run(command: list[str], **_kwargs: object) -> ExecResult:
+ captured_commands.append(command)
+ if command[:2] == ["container", "ls"]:
+ return ExecResult(
+ stdout=f"{identity}\n",
+ stderr="time=warning msg=obsolete compose version\n",
+ return_code=0,
+ )
+ return ExecResult(
+ stdout=(
+ f"{identity}\tsecure-compose-public-exec-test\thelper\t1\tFalse\t"
+ f"{'c' * 64}\ttrue\tfalse\tfalse\trunning\tnone\n"
+ ),
+ stderr="inspection warning\n",
+ return_code=0,
+ )
+
+ monkeypatch.setattr(environment, "_run_trusted_docker_command", run)
+
+ assert asyncio.run(environment._raw_service_container_ids("helper")) == (identity,)
+ assert captured_commands[0][:2] == ["container", "ls"]
+ assert "label=com.docker.compose.oneoff=False" in captured_commands[0]
+ assert "label=com.docker.compose.config-hash" in captured_commands[0]
+
+
+def test_raw_service_resolution_rejects_warning_or_malformed_stdout_before_action(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ captured_commands: list[list[str]] = []
+
+ async def run(command: list[str], **_kwargs: object) -> ExecResult:
+ captured_commands.append(command)
+ return ExecResult(
+ stdout="warning on stdout\n",
+ stderr=None,
+ return_code=0,
+ )
+
+ monkeypatch.setattr(environment, "_run_trusted_docker_command", run)
+
+ with pytest.raises(RuntimeError, match="invalid service container identity"):
+ asyncio.run(environment._raw_service_container_ids("helper"))
+
+ assert len(captured_commands) == 1
+ assert captured_commands[0][:2] == ["container", "ls"]
+
+
+@pytest.mark.parametrize(
+ "malformation",
+ ["reversed", "duplicate-number", "invalid-running", "wrong-service"],
+)
+def test_raw_service_resolution_rejects_malformed_or_mislabeled_inspect_state(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ malformation: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ identities = ("8" * 64, "9" * 64)
+ captured_commands: list[list[str]] = []
+
+ def state_line(identity: str, number: int) -> str:
+ service = "observer" if malformation == "wrong-service" and identity == identities[0] else "helper"
+ running = "unknown" if malformation == "invalid-running" and identity == identities[0] else "true"
+ if malformation == "duplicate-number":
+ number = 1
+ return (
+ f"{identity}\tsecure-compose-public-exec-test\t{service}\t{number}\tFalse\t"
+ f"{'d' * 64}\t{running}\tfalse\tfalse\trunning\tnone"
+ )
+
+ async def run(command: list[str], **_kwargs: object) -> ExecResult:
+ captured_commands.append(command)
+ if command[:2] == ["container", "ls"]:
+ return ExecResult(
+ stdout="\n".join(identities) + "\n",
+ stderr=None,
+ return_code=0,
+ )
+ lines = [state_line(identities[0], 1), state_line(identities[1], 2)]
+ if malformation == "reversed":
+ lines.reverse()
+ return ExecResult(
+ stdout="\n".join(lines) + "\n",
+ stderr=None,
+ return_code=0,
+ )
+
+ monkeypatch.setattr(environment, "_run_trusted_docker_command", run)
+
+ with pytest.raises(RuntimeError, match=r"invalid container state|duplicate service container numbers"):
+ asyncio.run(environment._raw_service_container_ids("helper"))
+
+ assert len(captured_commands) == 2
+ assert captured_commands[0][:2] == ["container", "ls"]
+ assert captured_commands[1][:2] == ["container", "inspect"]
+
+
+def test_same_sidecar_callback_reentry_fails_fast_without_deadlock(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ call_count = 0
+
+ async def emit_once(
+ _command: list[str],
+ on_output: object | None = None,
+ **_kwargs: object,
+ ) -> ExecResult:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ assert callable(on_output)
+ await on_output("outer-output\n", "stdout")
+ return ExecResult(stdout="ok", stderr=None, return_code=0)
+
+ async def reenter(_text: str, _stream: str) -> None:
+ await environment.service_exec("nested", service="helper")
+
+ monkeypatch.setattr(environment, "_run_docker_compose_command", emit_once)
+
+ async def exercise() -> None:
+ with (
+ environment.scoped_output_callback(reenter),
+ pytest.raises(RuntimeError, match=r"reentrant sidecar operation.*helper"),
+ ):
+ await asyncio.wait_for(
+ environment.service_exec("outer", service="helper"),
+ timeout=1,
+ )
+
+ asyncio.run(exercise())
+ assert call_count == 1
+
+
+@pytest.mark.parametrize("nested_operation", ["stop", "download-file", "download-dir"])
+def test_sidecar_callback_reentrant_lifecycle_operation_fails_fast_without_deadlock(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ nested_operation: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ call_count = 0
+
+ async def emit_once(
+ _command: list[str],
+ on_output: object | None = None,
+ **_kwargs: object,
+ ) -> ExecResult:
+ nonlocal call_count
+ call_count += 1
+ assert callable(on_output)
+ await on_output("outer-output\n", "stdout")
+ return ExecResult(stdout="ok", stderr=None, return_code=0)
+
+ async def reenter(_text: str, _stream: str) -> None:
+ if nested_operation == "stop":
+ await environment.stop_service("helper")
+ elif nested_operation == "download-file":
+ await environment.service_download_file(
+ "/tmp/source",
+ tmp_path / "target-file",
+ service="helper",
+ )
+ else:
+ await environment.service_download_dir(
+ "/tmp/source",
+ tmp_path / "target-dir",
+ service="helper",
+ )
+
+ monkeypatch.setattr(environment, "_run_docker_compose_command", emit_once)
+
+ async def exercise() -> None:
+ with (
+ environment.scoped_output_callback(reenter),
+ pytest.raises(RuntimeError, match=r"reentrant sidecar operation.*helper"),
+ ):
+ await asyncio.wait_for(
+ environment.service_exec("outer", service="helper"),
+ timeout=0.2,
+ )
+
+ asyncio.run(exercise())
+ assert call_count == 1
+
+
+def test_inactive_sidecar_reentry_marker_does_not_poison_callback_background_task(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ background_tasks: list[asyncio.Task[ExecResult]] = []
+ call_count = 0
+
+ async def emit_once(
+ _command: list[str],
+ on_output: object | None = None,
+ **_kwargs: object,
+ ) -> ExecResult:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ assert callable(on_output)
+ await on_output("outer-output\n", "stdout")
+ return ExecResult(stdout=f"call-{call_count}", stderr=None, return_code=0)
+
+ async def spawn_background(_text: str, _stream: str) -> None:
+ async def after_outer_finishes() -> ExecResult:
+ await asyncio.sleep(0.01)
+ return await environment.service_exec("later", service="helper")
+
+ background_tasks.append(asyncio.create_task(after_outer_finishes()))
+
+ monkeypatch.setattr(environment, "_run_docker_compose_command", emit_once)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(spawn_background):
+ outer = await environment.service_exec("outer", service="helper")
+ assert outer.stdout == "call-1"
+ assert len(background_tasks) == 1
+ return await asyncio.wait_for(background_tasks[0], timeout=1)
+
+ assert asyncio.run(exercise()).stdout == "call-2"
+
+
+def test_sidecar_compose_client_restores_operational_host_environment(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ audit_path = tmp_path / "sidecar-client-env.json"
+ bin_dir = tmp_path / "host-bin"
+ bin_dir.mkdir()
+ docker_path = bin_dir / "docker"
+ main_secret = "main-only-compose-client-secret-87235"
+ docker_path.write_text(
+ f"""#!{sys.executable}
+import json
+import os
+
+main_secret = {main_secret!r}
+with open({str(audit_path)!r}, "w", encoding="utf-8") as audit:
+ json.dump({{
+ "PATH": os.environ.get("PATH"),
+ "HOME": os.environ.get("HOME"),
+ "DOCKER_HOST": os.environ.get("DOCKER_HOST"),
+ "leaked_names": sorted(name for name in os.environ if name.startswith("MAIN_ONLY")),
+ "leaked_values": sorted(name for name, value in os.environ.items() if main_secret in value),
+ }}, audit)
+""",
+ encoding="utf-8",
+ )
+ docker_path.chmod(0o700)
+ host_home = str(tmp_path / "host-home")
+ host_docker = "unix:///safe-host-docker.sock"
+ monkeypatch.setenv("PATH", str(bin_dir))
+ monkeypatch.setenv("HOME", host_home)
+ monkeypatch.setenv("DOCKER_HOST", host_docker)
+ monkeypatch.setenv("MAIN_ONLY_WRAPPED_HOST", f"prefix:{main_secret}:suffix")
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={
+ "PATH": "/main-only-bin",
+ "HOME": "/main-only-home",
+ "DOCKER_HOST": "unix:///main-only-docker.sock",
+ "MAIN_ONLY_TOKEN": main_secret,
+ },
+ )
+
+ result = asyncio.run(environment.service_exec("true", service="helper"))
+ audit = json.loads(audit_path.read_text(encoding="utf-8"))
+
+ assert result.return_code == 0
+ assert audit == {
+ "PATH": str(bin_dir),
+ "HOME": host_home,
+ "DOCKER_HOST": host_docker,
+ "leaked_names": [],
+ "leaked_values": [],
+ }
+
+
+def test_raw_docker_host_baseline_retains_windows_home_controls_without_home(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ monkeypatch.delenv("HOME", raising=False)
+ windows_home = {
+ "USERPROFILE": r"C:\Users\trusted",
+ "HOMEDRIVE": "C:",
+ "HOMEPATH": r"\Users\trusted",
+ }
+ for name, value in windows_home.items():
+ monkeypatch.setenv(name, value)
+ monkeypatch.setenv("COMPOSE_FILE", r"C:\task-controlled\compose.yaml")
+
+ raw_environment = environment._trusted_docker_client_environment()
+
+ assert {name: raw_environment[name] for name in windows_home} == windows_home
+ assert "HOME" not in raw_environment
+ assert "COMPOSE_FILE" not in raw_environment
+
+
+def test_main_secure_handoff_restores_host_control_environment_for_every_compose_client(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ audit_path = tmp_path / "main-control-audit.jsonl"
+ bin_dir = tmp_path / "trusted-main-bin"
+ bin_dir.mkdir()
+ docker_path = bin_dir / "docker"
+ docker_path.write_text(
+ f"""#!{sys.executable}
+import json
+import os
+import sys
+
+with open({str(audit_path)!r}, "a", encoding="utf-8") as audit:
+ audit.write(json.dumps({{
+ "PATH": os.environ.get("PATH"),
+ "HOME": os.environ.get("HOME"),
+ "DOCKER_HOST": os.environ.get("DOCKER_HOST"),
+ "DOCKER_CONFIG": os.environ.get("DOCKER_CONFIG"),
+ "COMPOSE_FILE": os.environ.get("COMPOSE_FILE"),
+ "COMPOSE_ENV_FILES": os.environ.get("COMPOSE_ENV_FILES"),
+ "COMPOSE_DISABLE_ENV_FILE": os.environ.get("COMPOSE_DISABLE_ENV_FILE"),
+ }}) + "\\n")
+sys.stdin.buffer.read()
+""",
+ encoding="utf-8",
+ )
+ docker_path.chmod(0o700)
+ trusted_environment = {
+ "PATH": str(bin_dir),
+ "HOME": str(tmp_path / "trusted-main-home"),
+ "DOCKER_HOST": "unix:///trusted-main-docker.sock",
+ "DOCKER_CONFIG": str(tmp_path / "trusted-main-docker-config"),
+ "COMPOSE_FILE": str(tmp_path / "trusted-main-compose.yaml"),
+ }
+ target_environment = {
+ "PATH": "/main-target-bin",
+ "HOME": "/main-target-home",
+ "DOCKER_HOST": "tcp://main-target.invalid:2376",
+ "DOCKER_CONFIG": "/main-target-docker-config",
+ "COMPOSE_FILE": "/main-target-compose.yaml",
+ }
+ for name, value in trusted_environment.items():
+ monkeypatch.setenv(name, value)
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env=target_environment,
+ )
+
+ result = asyncio.run(environment.exec("true"))
+ audits = [json.loads(line) for line in audit_path.read_text(encoding="utf-8").splitlines()]
+ expected_client_environment = {
+ **trusted_environment,
+ "COMPOSE_FILE": None,
+ "COMPOSE_ENV_FILES": None,
+ "COMPOSE_DISABLE_ENV_FILE": "1",
+ }
+
+ assert result.return_code == 0
+ assert len(audits) == 4
+ assert all(audit == expected_client_environment for audit in audits)
+ assert all(
+ target_value not in value
+ for audit in audits
+ for value in audit.values()
+ for target_value in target_environment.values()
+ if value is not None
+ )
+
+
+def test_sidecar_filter_preserves_harbor_infra_over_user_name_collision(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ user_collision = "main-user-reserved-infra-secret"
+ environment._compose_task_env = {"MAIN_IMAGE_NAME": user_collision}
+ expected_infra_value = environment._compose_infra_env_vars()["MAIN_IMAGE_NAME"]
+ captured_environment: dict[str, str] = {}
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ captured_environment.update(kwargs["env"])
+ return _BufferedComposeProcess(stdout=b"")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ result = asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert result.return_code == 0
+ assert captured_environment["MAIN_IMAGE_NAME"] == expected_infra_value
+ assert all(user_collision not in value for value in captured_environment.values())
+
+
+def test_sidecar_compose_client_drops_unrelated_host_credentials(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ unrelated_credentials = {
+ "AWS_SECRET_ACCESS_KEY": "unrelated-host-aws-secret",
+ "GITHUB_TOKEN": "unrelated-host-github-token",
+ "OPENAI_API_KEY": "unrelated-host-openai-secret",
+ }
+ for name, value in unrelated_credentials.items():
+ monkeypatch.setenv(name, value)
+ captured_environment: dict[str, str] = {}
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ captured_environment.update(kwargs["env"])
+ return _BufferedComposeProcess(stdout=b"")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ result = asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert result.return_code == 0
+ assert not unrelated_credentials.keys() & captured_environment.keys()
+ assert all(
+ credential not in value
+ for credential in unrelated_credentials.values()
+ for value in captured_environment.values()
+ )
+
+
+def test_sidecar_retains_only_structurally_required_nonsecret_compose_task_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ compose_path = environment.environment_dir / "docker-compose.yaml"
+ compose_path.write_text(
+ "services:\n helper:\n image: ${HELPER_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ helper_image = "python:3.13-slim"
+ main_secret = "compose-model-main-api-secret"
+ monkeypatch.setenv(NVIDIA_BUILD_KEY_STDIN_ENV, "1")
+ environment._compose_task_env = {
+ "HELPER_IMAGE": helper_image,
+ "MAIN_API_TOKEN": main_secret,
+ "UNREFERENCED_SETTING": "not-needed-by-compose",
+ }
+ calls: list[tuple[tuple[str, ...], dict[str, str]]] = []
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedComposeProcess:
+ rendered = tuple(str(argument) for argument in args)
+ calls.append((rendered, dict(kwargs["env"])))
+ return _BufferedComposeProcess(stdout=b"sidecar-ok")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ result = asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert result.stdout == "sidecar-ok"
+ assert len(calls) == 1
+ exec_arguments, exec_environment = calls[0]
+ assert helper_image not in " ".join(exec_arguments)
+ assert exec_environment["HELPER_IMAGE"] == helper_image
+ assert "MAIN_API_TOKEN" not in exec_environment
+ assert "UNREFERENCED_SETTING" not in exec_environment
+ assert all(main_secret not in value for value in exec_environment.values())
+
+
+@pytest.mark.parametrize(
+ ("content", "expected"),
+ [
+ ("$PLAIN", {"PLAIN"}),
+ ("prefix ${BRACED} suffix", {"BRACED"}),
+ ("${DEFAULT:-fallback} ${REQUIRED:?required}", {"DEFAULT", "REQUIRED"}),
+ ("$$ESCAPED $${ALSO_ESCAPED}", set()),
+ ("${OUTER:-${INNER:-fallback}}", {"OUTER", "INNER"}),
+ ],
+)
+def test_compose_interpolation_scanner_handles_compose_forms(
+ content: str,
+ expected: set[str],
+) -> None:
+ assert _compose_interpolation_names(content) == expected
+
+
+def test_compose_model_parser_excludes_comments_and_mapping_keys_and_follows_safe_inputs(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ included = environment.environment_dir / "included.yaml"
+ extended = environment.environment_dir / "extended.yaml"
+ included.write_text(
+ "services:\n included:\n image: ${INCLUDED_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ extended.write_text(
+ "services:\n base:\n image: ${EXTENDED_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "# ${COMMENT_ONLY:?must-not-count}\n"
+ "include:\n - included.yaml\n"
+ "services:\n"
+ " helper:\n"
+ " extends:\n file: extended.yaml\n service: base\n"
+ " image: ${HELPER_IMAGE:?required}\n"
+ " labels:\n"
+ " ${LITERAL_MAPPING_KEY}: fixed\n"
+ " used: ${MAPPING_VALUE:?required}\n"
+ " equal-list: !override\n"
+ " - ${EQUAL_LIST_KEY}=value\n"
+ " volumes: !reset &tagged_values\n"
+ " - ${TAGGED_VALUE}:/data\n"
+ " anchor-user:\n"
+ " image: alpine:3.20\n"
+ " volumes: *tagged_values\n",
+ encoding="utf-8",
+ )
+
+ names = environment._compose_model_interpolation_names()
+
+ assert names >= {
+ "HELPER_IMAGE",
+ "MAPPING_VALUE",
+ "EQUAL_LIST_KEY",
+ "TAGGED_VALUE",
+ "INCLUDED_IMAGE",
+ "EXTENDED_IMAGE",
+ }
+ assert not {"COMMENT_ONLY", "LITERAL_MAPPING_KEY"} & names
+
+
+def test_docker_start_rejects_project_dotenv_before_parent_start(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ (environment.environment_dir / ".env").write_text(
+ "HELPER_IMAGE=alpine:3.20\n",
+ encoding="utf-8",
+ )
+ parent_started = False
+
+ async def parent_start(_self: object, force_build: bool) -> None:
+ nonlocal parent_started
+ assert force_build is False
+ parent_started = True
+
+ monkeypatch.setattr(DockerEnvironment, "start", parent_start)
+
+ with pytest.raises(RuntimeError, match=r"Docker Compose project \.env files are not supported"):
+ asyncio.run(environment.start(force_build=False))
+
+ assert parent_started is False
+
+
+def test_compose_model_parser_bounds_recursive_aliases_and_node_expansion(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ compose_path = environment.environment_dir / "docker-compose.yaml"
+ compose_path.write_text(
+ "x-loop: &loop\n - *loop\nservices:\n helper:\n image: ${HELPER_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+
+ assert "HELPER_IMAGE" in environment._compose_model_interpolation_names()
+
+ monkeypatch.setattr(secure_docker_environment, "_MAX_COMPOSE_MODEL_NODES", 1)
+ with pytest.raises(RuntimeError, match="could not inspect Docker Compose"):
+ environment._compose_model_interpolation_names()
+
+
+def test_compose_model_parser_rejects_non_regular_include_without_blocking(
+ tmp_path: Path,
+) -> None:
+ if not hasattr(os, "mkfifo"):
+ pytest.skip("named pipes are unavailable on this platform")
+ environment = _initialized_secure_docker_environment(tmp_path)
+ include_path = environment.environment_dir / "blocking-include.yaml"
+ os.mkfifo(include_path)
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "include:\n - blocking-include.yaml\nservices:\n helper:\n image: alpine:3.20\n",
+ encoding="utf-8",
+ )
+
+ started = time.monotonic()
+ with pytest.raises(RuntimeError, match="could not inspect Docker Compose"):
+ environment._compose_model_interpolation_names()
+
+ assert time.monotonic() - started < 1
+
+
+def test_compose_model_parser_rejects_include_dotenv_and_custom_project_directory(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ included_directory = environment.environment_dir / "included"
+ included_directory.mkdir()
+ (included_directory / "compose.yaml").write_text(
+ "services:\n helper:\n image: ${HELPER_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ (included_directory / ".env").write_text(
+ "API_TOKEN=must-not-enter-compose-client\n",
+ encoding="utf-8",
+ )
+ compose_path = environment.environment_dir / "docker-compose.yaml"
+ compose_path.write_text(
+ "include:\n - included/compose.yaml\nservices:\n main:\n image: alpine:3.20\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(RuntimeError, match="could not inspect Docker Compose"):
+ environment._compose_model_interpolation_names()
+
+ (included_directory / ".env").unlink()
+ compose_path.write_text(
+ "include:\n"
+ " - path: included/compose.yaml\n"
+ " project_directory: included\n"
+ "services:\n main:\n image: alpine:3.20\n",
+ encoding="utf-8",
+ )
+ with pytest.raises(RuntimeError, match="could not inspect Docker Compose"):
+ environment._compose_model_interpolation_names()
+
+
+def test_compose_model_parser_uses_project_directory_for_override_extends(
+ tmp_path: Path,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ override_directory = environment.environment_dir / "overrides"
+ override_directory.mkdir()
+ override_path = override_directory / "override.yaml"
+ override_path.write_text(
+ "services:\n helper:\n extends:\n file: common.yaml\n service: base\n",
+ encoding="utf-8",
+ )
+ (environment.environment_dir / "common.yaml").write_text(
+ "services:\n base:\n image: ${PROJECT_BASE_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ (override_directory / "common.yaml").write_text(
+ "services:\n base:\n image: ${SHADOW_IMAGE:?must-not-count}\n",
+ encoding="utf-8",
+ )
+ environment.extra_docker_compose_paths = [override_path]
+
+ names = environment._compose_model_interpolation_names()
+
+ assert "PROJECT_BASE_IMAGE" in names
+ assert "SHADOW_IMAGE" not in names
+
+
+def test_sidecar_retains_structurally_required_nonsecret_host_compose_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ compose_path = environment.environment_dir / "docker-compose.yaml"
+ compose_path.write_text(
+ "services:\n helper:\n image: alpine:3.20\n"
+ " ports:\n"
+ " - target: 80\n"
+ ' published: "${CUSTOM_PORT:?required}"\n'
+ ' host_ip: "${CUSTOM_HOST_IP:?required}"\n',
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("CUSTOM_PORT", "43127")
+ monkeypatch.setenv("CUSTOM_HOST_IP", "127.0.0.1")
+ captured_environments: list[dict[str, str]] = []
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ captured_environments.append(dict(kwargs["env"]))
+ return _BufferedComposeProcess(stdout=b"ok")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ result = asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert result.stdout == "ok"
+ assert len(captured_environments) == 1
+ assert captured_environments[0]["CUSTOM_PORT"] == "43127"
+ assert captured_environments[0]["CUSTOM_HOST_IP"] == "127.0.0.1"
+
+
+@pytest.mark.parametrize(
+ ("required_name", "required_value"),
+ [
+ ("API_TOKEN", "compose-required-api-token-secret"),
+ ("SIDECAR_API_KEY", "short7"),
+ ("DOCKER_HOST", "tcp://compose-target.invalid:2376"),
+ ("HELPER_IMAGE", "prefix:compose-wrapped-secret:suffix"),
+ ],
+)
+def test_sidecar_rejects_protected_compose_interpolation_before_user_command(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ required_name: str,
+ required_value: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ compose_path = environment.environment_dir / "docker-compose.yaml"
+ compose_path.write_text(
+ f"services:\n helper:\n image: ${{{required_name}:?required}}\n",
+ encoding="utf-8",
+ )
+ environment._compose_task_env = {required_name: required_value}
+ if required_name == "HELPER_IMAGE":
+ environment._compose_task_env["MAIN_API_TOKEN"] = "compose-wrapped-secret"
+ calls: list[tuple[tuple[str, ...], dict[str, str]]] = []
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedComposeProcess:
+ rendered = tuple(str(argument) for argument in args)
+ calls.append((rendered, dict(kwargs["env"])))
+ return _BufferedComposeProcess(stdout=b"must-not-spawn")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(
+ RuntimeError,
+ match=r"requires protected execution state|cannot (?:override|use) host client controls",
+ ) as caught:
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert calls == []
+ assert required_value not in str(caught.value)
+
+
+def test_sidecar_rejects_host_docker_auth_interpolation_before_spawn(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ docker_auth = "host-docker-auth-config-secret"
+ monkeypatch.setenv("DOCKER_AUTH_CONFIG", docker_auth)
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "services:\n helper:\n image: ${DOCKER_AUTH_CONFIG:?required}\n",
+ encoding="utf-8",
+ )
+ spawned = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> object:
+ nonlocal spawned
+ spawned = True
+ raise AssertionError("host Docker authorization must fail before spawn")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="requires protected execution state") as caught:
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert docker_auth not in str(caught.value)
+ assert spawned is False
+
+
+def test_sidecar_rejects_compose_value_wrapping_short_sensitive_main_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ short_secret = "x"
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"API_TOKEN": short_secret},
+ )
+ monkeypatch.setenv("HELPER_IMAGE", f"alpine:{short_secret}")
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "services:\n helper:\n image: ${HELPER_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ spawned = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> object:
+ nonlocal spawned
+ spawned = True
+ raise AssertionError("short sensitive wrapper must fail before spawn")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="requires protected execution state"):
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert spawned is False
+
+
+@pytest.mark.parametrize("wrapper", ["{}", "prefix:{}:suffix"])
+def test_sidecar_rejects_compose_value_reusing_other_main_only_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ wrapper: str,
+) -> None:
+ main_only_value = "non-sensitive-main-only-build-reference"
+ required_value = wrapper.format(main_only_value)
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"BUILD_REF": main_only_value},
+ )
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "services:\n helper:\n image: ${CUSTOM_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("CUSTOM_IMAGE", required_value)
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ raise AssertionError("protected Compose interpolation must fail before spawn")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="requires protected execution state") as caught:
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert main_only_value not in str(caught.value)
+ assert required_value not in str(caught.value)
+
+
+def test_sidecar_rejects_effective_compose_value_wrapping_shadowed_same_name_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ shadowed_value = "shadowed-same-name-main-only-value"
+ effective_value = f"prefix:{shadowed_value}:suffix"
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"BUILD_REF": effective_value},
+ )
+ environment._compose_task_env = {"BUILD_REF": shadowed_value}
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "services:\n helper:\n image: ${BUILD_REF:?required}\n",
+ encoding="utf-8",
+ )
+ spawned = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ nonlocal spawned
+ spawned = True
+ return _BufferedComposeProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="requires protected execution state") as caught:
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert shadowed_value not in str(caught.value)
+ assert effective_value not in str(caught.value)
+ assert spawned is False
+
+
+def test_sidecar_control_env_uses_carriers_without_redirecting_compose_client(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ audit_path = tmp_path / "sidecar-control-env.json"
+ bin_dir = tmp_path / "trusted-host-bin"
+ bin_dir.mkdir()
+ docker_path = bin_dir / "docker"
+ docker_path.write_text(
+ f"""#!{sys.executable}
+import json
+import os
+import sys
+
+with open({str(audit_path)!r}, "w", encoding="utf-8") as audit:
+ json.dump({{
+ "argv": sys.argv[1:],
+ "PATH": os.environ.get("PATH"),
+ "HOME": os.environ.get("HOME"),
+ "DOCKER_HOST": os.environ.get("DOCKER_HOST"),
+ "DOCKER_CONFIG": os.environ.get("DOCKER_CONFIG"),
+ "COMPOSE_FILE": os.environ.get("COMPOSE_FILE"),
+ "COMPOSE_ENV_FILES": os.environ.get("COMPOSE_ENV_FILES"),
+ "COMPOSE_DISABLE_ENV_FILE": os.environ.get("COMPOSE_DISABLE_ENV_FILE"),
+ "carriers": {{
+ name: value
+ for name, value in os.environ.items()
+ if name.startswith("SKILLEVALUATOR_SIDECAR_ENV_")
+ }},
+ }}, audit)
+""",
+ encoding="utf-8",
+ )
+ docker_path.chmod(0o700)
+ host_home = str(tmp_path / "trusted-host-home")
+ host_docker = "unix:///trusted-host-docker.sock"
+ host_docker_config = str(tmp_path / "trusted-docker-config")
+ host_compose_file = str(tmp_path / "trusted-compose.yaml")
+ monkeypatch.setenv("PATH", str(bin_dir))
+ monkeypatch.setenv("HOME", host_home)
+ monkeypatch.setenv("DOCKER_HOST", host_docker)
+ monkeypatch.setenv("DOCKER_CONFIG", host_docker_config)
+ monkeypatch.setenv("COMPOSE_FILE", host_compose_file)
+ compose_env_file = tmp_path / "hostile-compose.env"
+ compose_env_file.write_text("HOSTILE_TOKEN=must-not-be-loaded\n", encoding="utf-8")
+ monkeypatch.setenv("COMPOSE_ENV_FILES", str(compose_env_file))
+ environment = _initialized_secure_docker_environment(tmp_path)
+ target_environment = {
+ "PATH": "/sidecar-only-bin",
+ "HOME": "/sidecar-only-home",
+ "DOCKER_HOST": "tcp://sidecar-only.invalid:2376",
+ "DOCKER_CONFIG": "/sidecar-only-docker-config",
+ "COMPOSE_FILE": "/sidecar-only-compose.yaml",
+ "NORMAL_TOKEN": "sidecar 'quoted' $dollar\nsecond-line-secret",
+ "EMPTY_VALUE": "",
+ }
+
+ result = asyncio.run(
+ environment.service_exec(
+ "printf control-env",
+ service="helper",
+ env=target_environment,
+ )
+ )
+ audit = json.loads(audit_path.read_text(encoding="utf-8"))
+ rendered_argv = " ".join(audit["argv"])
+
+ assert result.return_code == 0
+ assert audit["PATH"] == str(bin_dir)
+ assert audit["HOME"] == host_home
+ assert audit["DOCKER_HOST"] == host_docker
+ assert audit["DOCKER_CONFIG"] == host_docker_config
+ assert audit["COMPOSE_FILE"] is None
+ assert audit["COMPOSE_ENV_FILES"] is None
+ assert audit["COMPOSE_DISABLE_ENV_FILE"] == "1"
+ assert set(audit["carriers"].values()) == set(target_environment.values())
+ assert len(audit["carriers"]) == len(target_environment)
+ assert all(target_value not in rendered_argv for target_value in target_environment.values() if target_value)
+ assert all(f"export {target_name}=" in rendered_argv for target_name in target_environment)
+ assert 'exec /bin/sh -c "$1"' in rendered_argv
+ assert audit["argv"][-2:] == ["sh", "printf control-env"]
+ assert all(target_name not in audit["carriers"] for target_name in target_environment)
+
+
+@pytest.mark.parametrize("collision_source", ["target", "base"])
+def test_sidecar_carriers_retry_target_and_base_environment_name_collisions(
+ monkeypatch: pytest.MonkeyPatch,
+ collision_source: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ first_id = "A" * 32
+ second_id = "B" * 32
+ first_carrier = f"SKILLEVALUATOR_SIDECAR_ENV_{first_id}_0"
+ second_carrier = f"SKILLEVALUATOR_SIDECAR_ENV_{second_id}_0"
+ generated_ids = iter((first_id, second_id))
+ monkeypatch.setattr(
+ secure_docker_environment.uuid,
+ "uuid4",
+ lambda: SimpleNamespace(hex=next(generated_ids)),
+ )
+ target_name = first_carrier if collision_source == "target" else "TARGET_TOKEN"
+ reserved_names = {first_carrier} if collision_source == "base" else set()
+
+ arguments, carriers, wrapper = _sidecar_environment_carriers(
+ {target_name: "collision-safe-sidecar-value"},
+ reserved_names=reserved_names,
+ )
+
+ assert arguments == ["-e", second_carrier]
+ assert carriers == {second_carrier: "collision-safe-sidecar-value"}
+ assert wrapper is not None
+ assert f"export {target_name}=" in wrapper
+ assert f"unset {second_carrier}" in wrapper
+
+
+@pytest.mark.parametrize("required_name", ["PATH", "DOCKER_HOST", "MAIN_IMAGE_NAME"])
+def test_sidecar_fails_closed_when_trusted_client_value_wraps_main_secret(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ required_name: str,
+) -> None:
+ secret = "protected-main-secret-inside-client-control"
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"MAIN_ONLY_TOKEN": secret},
+ )
+ spawned = False
+
+ if required_name == "MAIN_IMAGE_NAME":
+ infrastructure = environment._compose_infra_env_vars()
+
+ def poisoned_infrastructure() -> dict[str, str]:
+ return {
+ **infrastructure,
+ required_name: f"prefix:{secret}:suffix",
+ }
+
+ monkeypatch.setattr(environment, "_compose_infra_env_vars", poisoned_infrastructure)
+ else:
+ monkeypatch.setenv(required_name, f"prefix:{secret}:suffix")
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ nonlocal spawned
+ spawned = True
+ return _BufferedComposeProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="contains protected execution state") as caught:
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert required_name in str(caught.value)
+ assert secret not in str(caught.value)
+ assert spawned is False
+
+
+def test_sidecar_fails_closed_when_harbor_infra_wraps_main_stdin_sentinel(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ infrastructure = environment._compose_infra_env_vars()
+ spawned = False
+
+ monkeypatch.setattr(
+ environment,
+ "_compose_infra_env_vars",
+ lambda: {
+ **infrastructure,
+ "MAIN_IMAGE_NAME": f"prefix:{NVIDIA_BUILD_STDIN_SENTINEL}:suffix",
+ },
+ )
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ nonlocal spawned
+ spawned = True
+ return _BufferedComposeProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="contains protected execution state") as caught:
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert NVIDIA_BUILD_STDIN_SENTINEL not in str(caught.value)
+ assert spawned is False
+
+
+def test_service_stop_and_sidecar_downloads_scrub_main_compose_environment(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persistent_secret = "service-lifecycle-persistent-secret"
+ task_secret = "service-lifecycle-task-secret"
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"MAIN_ONLY_PERSISTENT": persistent_secret},
+ )
+ environment._compose_task_env = {"MAIN_ONLY_TASK": task_secret}
+ monkeypatch.setenv("SERVICE_LIFECYCLE_WRAPPED", f"prefix:{persistent_secret}:{task_secret}:suffix")
+ calls: list[tuple[tuple[str, ...], dict[str, str]]] = []
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedComposeProcess:
+ calls.append((tuple(str(argument) for argument in args), dict(kwargs["env"])))
+ return _BufferedComposeProcess(stdout=b"")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> None:
+ await environment.stop_service(MAIN_SERVICE_NAME)
+ await environment.service_download_file(
+ "/tmp/sidecar-file",
+ tmp_path / "downloaded-file",
+ service="helper",
+ )
+ await environment.service_download_dir(
+ "/tmp/sidecar-dir",
+ tmp_path / "downloaded-dir",
+ service="helper",
+ )
+
+ asyncio.run(exercise())
+
+ assert len(calls) == 3
+ assert calls[0][0][1:3] == ("container", "ls")
+ assert "label=com.docker.compose.service=main" in calls[0][0]
+ assert calls[1][0][-4:] == (
+ "cp",
+ "--",
+ "helper:/tmp/sidecar-file",
+ str(tmp_path / "downloaded-file"),
+ )
+ assert calls[2][0][-4:] == (
+ "cp",
+ "--",
+ "helper:/tmp/sidecar-dir/.",
+ str(tmp_path / "downloaded-dir"),
+ )
+ for _arguments, process_environment in calls:
+ assert (
+ not {
+ "MAIN_ONLY_PERSISTENT",
+ "MAIN_ONLY_TASK",
+ "SERVICE_LIFECYCLE_WRAPPED",
+ }
+ & process_environment.keys()
+ )
+ assert all(
+ secret not in value for value in process_environment.values() for secret in (persistent_secret, task_secret)
+ )
+
+
+@pytest.mark.parametrize("target_service", [MAIN_SERVICE_NAME, "helper"])
+def test_service_stop_uses_label_scoped_raw_docker_when_compose_model_requires_secret(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ target_service: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ protected_value = "protected-stop-api-token"
+ trusted_host_controls = {
+ "DOCKER_HOST": "unix:///trusted-host-docker.sock",
+ "DOCKER_CONFIG": str(tmp_path / "trusted-host-docker-config"),
+ }
+ for name, value in trusted_host_controls.items():
+ monkeypatch.setenv(name, value)
+ monkeypatch.setenv("COMPOSE_FILE", "/trusted/compose-must-not-reach-raw-docker.yaml")
+ environment._compose_task_env = {
+ "API_TOKEN": protected_value,
+ "PATH": "/task-controlled-path",
+ "HOME": "/task-controlled-home",
+ "DOCKER_HOST": "tcp://task-controlled.invalid:2376",
+ "DOCKER_CONFIG": "/task-controlled-docker-config",
+ "COMPOSE_FILE": "/task-controlled-compose.yaml",
+ "PATH_OVERLAP": os.environ["PATH"],
+ }
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "services:\n"
+ " main:\n image: alpine:3.20\n"
+ " environment:\n API_TOKEN: ${API_TOKEN:?required}\n"
+ " helper:\n image: alpine:3.20\n",
+ encoding="utf-8",
+ )
+ project = "secure-compose-public-exec-test"
+ main_id = "a" * 64
+ helper_id = "b" * 64
+ containers = {
+ main_id: {"project": project, "service": "main", "number": 1, "running": True},
+ helper_id: {"project": project, "service": "helper", "number": 1, "running": True},
+ }
+ calls: list[tuple[tuple[str, ...], dict[str, str]]] = []
+
+ class RawDockerProcess:
+ pid = 8123
+
+ def __init__(self, arguments: tuple[str, ...]) -> None:
+ self.arguments = arguments
+ self.returncode: int | None = 0
+
+ async def communicate(self, **_kwargs: bytes | None) -> tuple[bytes, bytes]:
+ arguments = list(self.arguments[1:])
+ if arguments[:2] == ["container", "ls"]:
+ filters = [arguments[index + 1] for index, value in enumerate(arguments) if value == "--filter"]
+ project_filter = next(value.split("=", 2)[2] for value in filters if ".project=" in value)
+ service_filter = next(value.split("=", 2)[2] for value in filters if ".service=" in value)
+ matches = [
+ container_id
+ for container_id, state in containers.items()
+ if state["project"] == project_filter and state["service"] == service_filter
+ ]
+ stdout = "\n".join(matches) + ("\n" if matches else "")
+ return stdout.encode(), b"warning on stderr"
+ if arguments[:2] == ["container", "inspect"]:
+ identities = arguments[arguments.index("--") + 1 :]
+ states = [
+ "\t".join(
+ (
+ identity,
+ str(containers[identity]["project"]),
+ str(containers[identity]["service"]),
+ str(containers[identity]["number"]),
+ "False",
+ "c" * 64,
+ str(containers[identity]["running"]).lower(),
+ "false",
+ "false",
+ "running" if containers[identity]["running"] else "exited",
+ "none",
+ )
+ )
+ for identity in identities
+ if identity in containers
+ ]
+ if len(states) != len(identities):
+ self.returncode = 1
+ stdout = "\n".join(states) + ("\n" if states else "")
+ return stdout.encode(), b""
+ if arguments[:2] in (["container", "stop"], ["container", "kill"]):
+ for identity in arguments[arguments.index("--") + 1 :]:
+ containers[identity]["running"] = False
+ return b"", b""
+ raise AssertionError(f"unexpected raw Docker command: {arguments!r}")
+
+ def terminate(self) -> None:
+ self.returncode = -signal.SIGTERM
+
+ def kill(self) -> None:
+ self.returncode = -signal.SIGKILL
+
+ async def create_subprocess(*args: object, **kwargs: object) -> RawDockerProcess:
+ rendered = tuple(str(argument) for argument in args)
+ process_environment = dict(kwargs["env"])
+ calls.append((rendered, process_environment))
+ assert "compose" not in rendered
+ assert protected_value not in process_environment.values()
+ assert "COMPOSE_FILE" not in process_environment
+ assert process_environment["PATH"] == os.environ["PATH"]
+ assert process_environment["HOME"] == os.environ["HOME"]
+ assert {name: process_environment[name] for name in trusted_host_controls} == trusted_host_controls
+ return RawDockerProcess(rendered)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ asyncio.run(environment.stop_service(target_service))
+
+ assert containers[main_id]["running"] is (target_service != MAIN_SERVICE_NAME)
+ assert containers[helper_id]["running"] is (target_service != "helper")
+ assert calls
+ assert all(
+ f"label=com.docker.compose.project={project}" in arguments for arguments, _env in calls if "ls" in arguments
+ )
+ assert all(
+ f"label=com.docker.compose.service={target_service}" in arguments
+ for arguments, _env in calls
+ if "ls" in arguments
+ )
+ assert all(
+ "label=com.docker.compose.oneoff=False" in arguments and "label=com.docker.compose.config-hash" in arguments
+ for arguments, _env in calls
+ if "ls" in arguments
+ )
+
+
+def test_service_stop_rejects_unknown_service_before_raw_docker_spawn(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ spawned = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> object:
+ nonlocal spawned
+ spawned = True
+ raise AssertionError("unknown service must fail before spawn")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="unknown Docker Compose service 'does-not-exist'"):
+ asyncio.run(environment.stop_service("does-not-exist"))
+
+ assert spawned is False
+
+
+def test_service_stop_accepts_service_declared_by_trusted_compose_include(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ (environment.environment_dir / "included.yaml").write_text(
+ "services:\n included-helper:\n image: alpine:3.20\n",
+ encoding="utf-8",
+ )
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "include:\n - included.yaml\nservices:\n helper:\n image: alpine:3.20\n",
+ encoding="utf-8",
+ )
+ calls: list[tuple[object, ...]] = []
+
+ async def create_subprocess(*args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ calls.append(args)
+ return _BufferedComposeProcess(stdout=b"")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ asyncio.run(environment.stop_service("included-helper"))
+
+ assert len(calls) == 1
+ assert "label=com.docker.compose.service=included-helper" in calls[0]
+
+
+def test_raw_sidecar_containment_kills_survivor_and_restores_only_running_snapshot(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ project = "secure-compose-public-exec-test"
+ first_id, second_id, initially_stopped_id = ("1" * 64, "2" * 64, "3" * 64)
+ containers: dict[str, dict[str, object]] = {
+ first_id: {"number": 1, "running": True, "health": None},
+ second_id: {"number": 2, "running": True, "health": "healthy"},
+ initially_stopped_id: {"number": 3, "running": False, "health": None},
+ }
+ actions: list[tuple[tuple[str, ...], tuple[str, ...]]] = []
+ restore_poll_count = 0
+ restoration_started = False
+
+ async def ids(service: str) -> tuple[str, ...]:
+ assert service == "helper"
+ return first_id, second_id, initially_stopped_id
+
+ async def states(
+ identities: tuple[str, ...],
+ *,
+ service: str,
+ ) -> dict[str, object]:
+ nonlocal restore_poll_count
+ assert service == "helper"
+ if restoration_started and identities == (
+ first_id,
+ second_id,
+ initially_stopped_id,
+ ):
+ restore_poll_count += 1
+ rendered: dict[str, object] = {}
+ for identity in identities:
+ state = containers[identity]
+ running = bool(state["running"])
+ health = state["health"]
+ if identity == second_id and restoration_started and running:
+ health = "starting" if restore_poll_count == 1 else "healthy"
+ rendered[identity] = secure_docker_environment._RawContainerState(
+ identity=identity,
+ project=project,
+ service=service,
+ container_number=int(state["number"]),
+ running=running,
+ paused=False,
+ restarting=False,
+ status="running" if running else "exited",
+ health_status=health,
+ )
+ return rendered
+
+ async def action(command: list[str], identities: tuple[str, ...]) -> bool:
+ nonlocal restoration_started
+ actions.append((tuple(command), tuple(identities)))
+ if command[:2] == ["container", "stop"]:
+ # Model a stop race: replica 1 survives and must be killed.
+ containers[second_id]["running"] = False
+ elif command[:2] == ["container", "kill"]:
+ for identity in identities:
+ containers[identity]["running"] = False
+ elif command[:2] == ["container", "start"]:
+ restoration_started = True
+ for identity in identities:
+ containers[identity]["running"] = True
+ else:
+ raise AssertionError(f"unexpected raw action: {command!r}")
+ return True
+
+ monkeypatch.setattr(environment, "_raw_service_container_ids", ids)
+ monkeypatch.setattr(environment, "_raw_container_states", states)
+ monkeypatch.setattr(environment, "_raw_docker_action", action)
+
+ async def exercise() -> tuple[object, bool]:
+ with secure_docker_environment._raw_lifecycle_deadline_scope():
+ snapshot = await environment._contain_sidecar_service("helper")
+ assert containers[initially_stopped_id]["running"] is False
+ restored = await environment._restore_sidecar_service(
+ "helper",
+ snapshot=snapshot,
+ )
+ return snapshot, restored
+
+ snapshot, restored = asyncio.run(exercise())
+
+ assert snapshot == secure_docker_environment._RawServiceSnapshot(
+ all_identities=(first_id, second_id, initially_stopped_id),
+ running_identities=(first_id, second_id),
+ )
+ assert restored is True
+ assert actions == [
+ (("container", "stop", "--timeout", "0"), (first_id, second_id)),
+ (("container", "kill", "--signal", "SIGKILL"), (first_id,)),
+ (("container", "start"), (first_id, second_id)),
+ ]
+ assert containers[initially_stopped_id]["running"] is False
+ assert restore_poll_count == 2
+
+
+def test_main_containment_uses_rm_fallback_when_state_inspection_fails(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ identity = "4" * 64
+ removed = False
+ actions: list[tuple[tuple[str, ...], tuple[str, ...]]] = []
+
+ async def validated_ids(_service: str) -> tuple[str, ...]:
+ raise RuntimeError("malformed Docker state")
+
+ async def filtered_ids(service: str) -> tuple[str, ...]:
+ assert service == MAIN_SERVICE_NAME
+ return () if removed else (identity,)
+
+ async def action(command: list[str], identities: set[str]) -> bool:
+ nonlocal removed
+ actions.append((tuple(command), tuple(sorted(identities))))
+ assert command == ["container", "rm", "--force", "--volumes"]
+ removed = True
+ return True
+
+ monkeypatch.setattr(environment, "_raw_service_container_ids", validated_ids)
+ monkeypatch.setattr(environment, "_raw_filtered_service_container_ids", filtered_ids)
+ monkeypatch.setattr(environment, "_raw_docker_action", action)
+
+ asyncio.run(environment._contain_main_container())
+
+ assert removed is True
+ assert actions == [
+ (("container", "rm", "--force", "--volumes"), (identity,)),
+ ]
+
+
+def test_sidecar_restore_rejects_replacement_container_before_start(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ original_id = "5" * 64
+ initially_stopped_id = "6" * 64
+ replacement_id = "7" * 64
+ snapshot = secure_docker_environment._RawServiceSnapshot(
+ all_identities=(original_id, initially_stopped_id),
+ running_identities=(original_id,),
+ )
+ action_called = False
+
+ async def ids(_service: str) -> tuple[str, ...]:
+ return original_id, initially_stopped_id, replacement_id
+
+ async def action(_command: list[str], _identities: tuple[str, ...]) -> bool:
+ nonlocal action_called
+ action_called = True
+ return True
+
+ monkeypatch.setattr(environment, "_raw_service_container_ids", ids)
+ monkeypatch.setattr(environment, "_raw_docker_action", action)
+
+ assert (
+ asyncio.run(
+ environment._restore_sidecar_service(
+ "helper",
+ snapshot=snapshot,
+ )
+ )
+ is False
+ )
+ assert action_called is False
+
+
+def test_main_service_downloads_restore_host_controls_and_harbor_infrastructure(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ audit_path = tmp_path / "main-download-client-audit.jsonl"
+ bin_dir = tmp_path / "trusted-download-bin"
+ bin_dir.mkdir()
+ docker_path = bin_dir / "docker"
+ docker_path.write_text(
+ f"""#!{sys.executable}
+import json
+import os
+
+with open({str(audit_path)!r}, "a", encoding="utf-8") as audit:
+ audit.write(json.dumps({{
+ "PATH": os.environ.get("PATH"),
+ "HOME": os.environ.get("HOME"),
+ "DOCKER_HOST": os.environ.get("DOCKER_HOST"),
+ "MAIN_IMAGE_NAME": os.environ.get("MAIN_IMAGE_NAME"),
+ "HARBOR_CONTAINER_NAME": os.environ.get("HARBOR_CONTAINER_NAME"),
+ }}) + "\\n")
+""",
+ encoding="utf-8",
+ )
+ docker_path.chmod(0o700)
+ trusted_controls = {
+ "PATH": str(bin_dir),
+ "HOME": str(tmp_path / "trusted-download-home"),
+ "DOCKER_HOST": "unix:///trusted-download-docker.sock",
+ }
+ for name, value in trusted_controls.items():
+ monkeypatch.setenv(name, value)
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={
+ "PATH": "/main-download-target-bin",
+ "HOME": "/main-download-target-home",
+ "DOCKER_HOST": "tcp://main-download-target.invalid:2376",
+ "MAIN_IMAGE_NAME": "main-download-user-infra-collision",
+ "HARBOR_CONTAINER_NAME": "main-download-user-harbor-collision",
+ },
+ )
+ environment._windows_container_name = "trusted-harbor-container-name"
+ trusted_infrastructure = environment._compose_infra_env_vars()["MAIN_IMAGE_NAME"]
+
+ async def exercise() -> None:
+ await environment.service_download_file(
+ "/tmp/source-file",
+ tmp_path / "target-file",
+ )
+ await environment.service_download_dir(
+ "/tmp/source-dir",
+ tmp_path / "target-dir",
+ service=MAIN_SERVICE_NAME,
+ )
+
+ asyncio.run(exercise())
+ audits = [json.loads(line) for line in audit_path.read_text(encoding="utf-8").splitlines()]
+
+ assert len(audits) == 2
+ assert all(
+ audit
+ == {
+ **trusted_controls,
+ "MAIN_IMAGE_NAME": trusted_infrastructure,
+ "HARBOR_CONTAINER_NAME": "trusted-harbor-container-name",
+ }
+ for audit in audits
+ )
+
+
+def test_windows_main_downloads_use_scrubbed_docker_env_and_validated_archives(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ main_secret = "windows-main-transfer-secret-value"
+ unrelated_secret = "windows-unrelated-host-secret-value"
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"MAIN_TOKEN": main_secret},
+ )
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ monkeypatch.setenv("UNRELATED_API_TOKEN", unrelated_secret)
+ calls: list[tuple[tuple[str, ...], dict[str, str]]] = []
+ docker_call_count = 0
+
+ class TransferProcess:
+ pid = 7643
+
+ def __init__(
+ self,
+ arguments: tuple[str, ...],
+ process_environment: dict[str, str],
+ archive_payload: bytes,
+ ) -> None:
+ self.arguments = arguments
+ self.process_environment = process_environment
+ self.returncode: int | None = None
+ self.stdout = _ChunkStream([archive_payload])
+ self.stderr = _ChunkStream([b"successful docker warning must not enter tar bytes"])
+
+ async def wait(self) -> int:
+ self.returncode = 0
+ return self.returncode
+
+ async def communicate(self, **_kwargs: bytes | None) -> tuple[bytes, bytes]:
+ raise AssertionError("Windows container archives must not be buffered by communicate()")
+
+ async def create_subprocess(*args: object, **kwargs: object) -> TransferProcess:
+ nonlocal docker_call_count
+ assert "env" in kwargs
+ assert Path(str(args[0])).name == "docker"
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+ docker_call_count += 1
+ archive_buffer = io.BytesIO()
+ with tarfile.open(fileobj=archive_buffer, mode="w") as archive:
+ if docker_call_count == 1:
+ payload = b"\x00windows-file\xff"
+ member = tarfile.TarInfo("./payload.bin")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ else:
+ directory = tarfile.TarInfo("./nested")
+ directory.type = tarfile.DIRTYPE
+ archive.addfile(directory)
+ payload = b"\x00windows-dir\xfe"
+ member = tarfile.TarInfo("./nested/value.bin")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ archive_payload = archive_buffer.getvalue()
+ arguments = tuple(str(argument) for argument in args)
+ process_environment = dict(kwargs["env"])
+ calls.append((arguments, process_environment))
+ return TransferProcess(
+ arguments,
+ process_environment,
+ archive_payload,
+ )
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ downloaded_file = tmp_path / "downloaded.bin"
+ downloaded_dir = tmp_path / "downloaded-dir"
+ downloaded_file.write_bytes(b"old-file")
+ (downloaded_dir / "nested").mkdir(parents=True)
+ (downloaded_dir / "nested" / "value.bin").write_bytes(b"old-directory-file")
+ (downloaded_dir / "preserved.bin").write_bytes(b"preserved")
+
+ async def exercise() -> None:
+ await environment.service_download_file(
+ "/remote/payload.bin",
+ downloaded_file,
+ )
+ await environment.service_download_dir(
+ "/remote/tree",
+ downloaded_dir,
+ service=MAIN_SERVICE_NAME,
+ )
+
+ asyncio.run(exercise())
+
+ assert downloaded_file.read_bytes() == b"\x00windows-file\xff"
+ assert (downloaded_dir / "nested" / "value.bin").read_bytes() == b"\x00windows-dir\xfe"
+ assert (downloaded_dir / "preserved.bin").read_bytes() == b"preserved"
+ assert len(calls) == 2
+ assert [Path(arguments[0]).name for arguments, _env in calls] == [
+ "docker",
+ "docker",
+ ]
+ assert calls[0][0][1:5] == (
+ "exec",
+ "--",
+ "harbor-secure-windows",
+ "tar",
+ )
+ assert calls[0][0][-4:] == ("-C", "/remote", "--", "payload.bin")
+ assert calls[1][0][-4:] == ("-C", "/remote/tree", "--", ".")
+ for _arguments, process_environment in calls:
+ assert "MAIN_TOKEN" not in process_environment
+ assert "UNRELATED_API_TOKEN" not in process_environment
+ assert all(
+ secret not in value for secret in (main_secret, unrelated_secret) for value in process_environment.values()
+ )
+
+
+@pytest.mark.parametrize(
+ ("source_path", "expected_parent"),
+ (
+ ("C:/payload.bin", "C:/"),
+ (r"C:\payload.bin", "C:/"),
+ ("/payload.bin", "/"),
+ ("payload.bin", "."),
+ ),
+)
+def test_windows_file_download_preserves_container_source_root_semantics(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ source_path: str,
+ expected_parent: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ calls: list[tuple[str, tuple[str, ...]]] = []
+
+ async def download_dir(
+ source_dir: str,
+ target_dir: Path | str,
+ **kwargs: object,
+ ) -> None:
+ archive_members = kwargs["archive_members"]
+ assert isinstance(archive_members, tuple)
+ calls.append((source_dir, archive_members))
+ (Path(target_dir) / "payload.bin").write_bytes(b"payload")
+
+ monkeypatch.setattr(environment, "_secure_windows_download_dir", download_dir)
+ target = tmp_path / "downloaded.bin"
+
+ asyncio.run(environment.service_download_file(source_path, target))
+
+ assert calls == [(expected_parent, ("payload.bin",))]
+ assert target.read_bytes() == b"payload"
+
+
+@pytest.mark.parametrize("source_path", ["", "C:/", "C:\\", "/", "bad\x00name"])
+def test_windows_file_download_rejects_invalid_container_source_path(
+ tmp_path: Path,
+ source_path: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+
+ with pytest.raises(RuntimeError, match="download path is invalid"):
+ asyncio.run(environment.service_download_file(source_path, tmp_path / "target.bin"))
+
+
+def test_windows_transfer_streams_large_binary_output_without_communicate(tmp_path: Path) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "large-transfer.bin"
+ output_size = 16 * 1024 * 1024 + 137
+
+ tracemalloc.start()
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ [
+ sys.executable,
+ "-c",
+ f"import sys; sys.stdout.buffer.write(bytes(range(256)) * {output_size // 256} + "
+ f"bytes(range({output_size % 256})))",
+ ],
+ process_environment=dict(os.environ),
+ protected_values=set(),
+ output_path=output_path,
+ idle_timeout_sec=5,
+ )
+ )
+ _current_bytes, peak_bytes = tracemalloc.get_traced_memory()
+ tracemalloc.stop()
+
+ observed_digest = hashlib.sha256()
+ with output_path.open("rb") as output:
+ while chunk := output.read(1024 * 1024):
+ observed_digest.update(chunk)
+ expected_digest = hashlib.sha256()
+ for _ in range(output_size // 256):
+ expected_digest.update(bytes(range(256)))
+ expected_digest.update(bytes(range(output_size % 256)))
+
+ assert output_path.stat().st_size == output_size
+ assert observed_digest.hexdigest() == expected_digest.hexdigest()
+ assert peak_bytes < 12 * 1024 * 1024
+
+
+def test_windows_transfer_removes_output_if_diagnostic_spool_creation_fails(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "unstarted-transfer.tar"
+ spawned = False
+
+ def fail_temporary_file(**_kwargs: object) -> object:
+ raise OSError("injected diagnostic spool failure")
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> object:
+ nonlocal spawned
+ spawned = True
+ raise AssertionError
+
+ monkeypatch.setattr(secure_docker_environment.tempfile, "TemporaryFile", fail_temporary_file)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(OSError, match="injected diagnostic spool failure"):
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ )
+ )
+
+ assert spawned is False
+ assert not output_path.exists()
+
+
+@pytest.mark.parametrize(
+ ("stdout_bytes", "stderr_bytes", "should_fail"),
+ (
+ (11, 0, True),
+ (0, 11, True),
+ (6, 5, True),
+ (6, 4, False),
+ ),
+)
+def test_windows_transfer_enforces_combined_temporary_disk_budget(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ stdout_bytes: int,
+ stderr_bytes: int,
+ should_fail: bool,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "budgeted-transfer.tar"
+
+ class TransferProcess:
+ pid = 7648
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _ChunkStream([b"o" * stdout_bytes])
+ self.stderr = _ChunkStream([b"e" * stderr_bytes])
+
+ async def wait(self) -> int:
+ self.returncode = 0
+ return self.returncode
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> TransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+ return TransferProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_DISK_RESERVE_BYTES", 0)
+ monkeypatch.setattr(
+ secure_docker_environment.shutil,
+ "disk_usage",
+ lambda _path: SimpleNamespace(free=10),
+ )
+
+ async def transfer() -> None:
+ await environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ )
+
+ if should_fail:
+ with pytest.raises(RuntimeError, match="exceeded its temporary disk budget"):
+ asyncio.run(transfer())
+ assert not output_path.exists()
+ else:
+ asyncio.run(transfer())
+ assert output_path.read_bytes() == b"o" * stdout_bytes
+
+
+def test_windows_artifact_disk_reserve_does_not_ratchet_between_phases(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ initial_free = 2 * 1024 * 1024 * 1024
+ later_free = 1024 * 1024 * 1024
+ free_values = iter((initial_free, later_free, later_free))
+ monkeypatch.setattr(
+ secure_docker_environment.shutil,
+ "disk_usage",
+ lambda _path: SimpleNamespace(free=next(free_values)),
+ )
+
+ reserve = secure_docker_environment._windows_artifact_filesystem_reserve(tmp_path)
+ assert reserve.minimum_free_bytes == initial_free // 20
+ exact_later_budget = later_free - reserve.minimum_free_bytes
+
+ secure_docker_environment._require_windows_artifact_resources(
+ tmp_path,
+ exact_later_budget,
+ required_entries=0,
+ minimum_free_bytes=reserve.minimum_free_bytes,
+ purpose="test phase",
+ )
+ with pytest.raises(RuntimeError, match="insufficient disk space"):
+ secure_docker_environment._require_windows_artifact_resources(
+ tmp_path,
+ exact_later_budget + 1,
+ required_entries=0,
+ minimum_free_bytes=reserve.minimum_free_bytes,
+ purpose="test phase",
+ )
+
+
+@pytest.mark.parametrize(
+ ("same_filesystem", "expected_publication_reserve"),
+ ((True, 20), (False, 10)),
+)
+def test_windows_download_reuses_or_separates_filesystem_reserves(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ same_filesystem: bool,
+ expected_publication_reserve: int,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ target = tmp_path / "target"
+ captured: dict[str, int] = {}
+ reserves = iter(
+ (
+ secure_docker_environment._WindowsFilesystemReserve(identity=1, minimum_free_bytes=10),
+ secure_docker_environment._WindowsFilesystemReserve(
+ identity=1 if same_filesystem else 2,
+ minimum_free_bytes=20,
+ ),
+ )
+ )
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ captured["transfer"] = int(kwargs["minimum_free_bytes"]) # type: ignore[arg-type]
+ Path(kwargs["output_path"]).write_bytes(b"unused") # type: ignore[arg-type]
+
+ def extract(_archive: Path, _target: Path, **kwargs: object) -> object:
+ captured["extraction"] = int(kwargs["minimum_free_bytes"]) # type: ignore[arg-type]
+ return secure_docker_environment._WindowsArtifactUsage(file_bytes=0, entries=1)
+
+ def require_resources(_path: Path, _required_bytes: int, **kwargs: object) -> None:
+ captured["publication"] = int(kwargs["minimum_free_bytes"]) # type: ignore[arg-type]
+
+ monkeypatch.setattr(secure_docker_environment, "_windows_artifact_filesystem_reserve", lambda _path: next(reserves))
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ monkeypatch.setattr(secure_docker_environment, "_extract_regular_tar_archive", extract)
+ monkeypatch.setattr(secure_docker_environment, "_require_windows_artifact_resources", require_resources)
+ monkeypatch.setattr(secure_docker_environment, "copytree_secure", lambda *_args, **_kwargs: None)
+
+ asyncio.run(environment.service_download_dir("C:/artifacts", target))
+
+ assert captured == {
+ "transfer": 20,
+ "extraction": 20,
+ "publication": expected_publication_reserve,
+ }
+
+
+def test_windows_large_file_download_extracts_and_publishes_with_bounded_memory(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ payload_path = tmp_path / "payload.bin"
+ payload_size = 16 * 1024 * 1024 + 113
+ block = bytes(range(256)) * 4096
+ with payload_path.open("wb") as payload_file:
+ remaining = payload_size
+ while remaining:
+ chunk = block[: min(remaining, len(block))]
+ payload_file.write(chunk)
+ remaining -= len(chunk)
+ archive_path = tmp_path / "source.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ archive.add(payload_path, arcname="payload.bin", recursive=False)
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ shutil.copyfile(archive_path, Path(kwargs["output_path"])) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ downloaded = tmp_path / "downloaded.bin"
+ downloaded.write_bytes(b"old")
+
+ tracemalloc.start()
+ asyncio.run(environment.service_download_file("/remote/payload.bin", downloaded))
+ _current_bytes, peak_bytes = tracemalloc.get_traced_memory()
+ tracemalloc.stop()
+
+ source_digest = hashlib.sha256()
+ downloaded_digest = hashlib.sha256()
+ with payload_path.open("rb") as source, downloaded.open("rb") as destination:
+ while chunk := source.read(1024 * 1024):
+ source_digest.update(chunk)
+ while chunk := destination.read(1024 * 1024):
+ downloaded_digest.update(chunk)
+
+ assert downloaded.stat().st_size == payload_size
+ assert downloaded_digest.hexdigest() == source_digest.hexdigest()
+ assert peak_bytes < 12 * 1024 * 1024
+
+
+@pytest.mark.parametrize("operation", ["file", "directory"])
+def test_windows_download_rejects_insufficient_publication_filesystem_before_target_mutation(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ operation: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ archive_buffer = io.BytesIO()
+ with tarfile.open(fileobj=archive_buffer, mode="w") as archive:
+ payload = b"new"
+ member = tarfile.TarInfo("./payload.bin")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ archive_payload = archive_buffer.getvalue()
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_ENTRY_DISK_BYTES", 1)
+ target = tmp_path / ("downloaded.bin" if operation == "file" else "downloaded-dir")
+ if operation == "file":
+ target.write_bytes(b"old")
+ target_budget = 7 # new stage (4) plus existing rollback (4), minus one
+ else:
+ target.mkdir()
+ (target / "sentinel.bin").write_bytes(b"old")
+ target_budget = 14 # new tree (5) plus two existing snapshots (2 * 5), minus one
+
+ def disk_budget(path: Path) -> tuple[int, int]:
+ if Path(path) == target.parent:
+ return target_budget, target_budget
+ return 1024 * 1024, 1024 * 1024
+
+ monkeypatch.setattr(secure_docker_environment, "_windows_artifact_disk_budget", disk_budget)
+
+ if operation == "file":
+ with pytest.raises(RuntimeError, match=r"insufficient disk space.*file publication"):
+ asyncio.run(environment.service_download_file("/remote/payload.bin", target))
+ assert target.read_bytes() == b"old"
+ else:
+ with pytest.raises(RuntimeError, match=r"insufficient disk space.*directory publication"):
+ asyncio.run(environment.service_download_dir("/remote/tree", target))
+ assert [child.name for child in target.iterdir()] == ["sentinel.bin"]
+ assert (target / "sentinel.bin").read_bytes() == b"old"
+
+
+def test_windows_file_publication_charges_missing_destination_parents(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ archive_buffer = io.BytesIO()
+ with tarfile.open(fileobj=archive_buffer, mode="w") as archive:
+ payload = b"new"
+ member = tarfile.TarInfo("./payload.bin")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ archive_payload = archive_buffer.getvalue()
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_ENTRY_DISK_BYTES", 1)
+ target = tmp_path / "missing-one" / "missing-two" / "downloaded.bin"
+
+ def disk_budget(path: Path) -> tuple[int, int]:
+ if Path(path) == target.parent:
+ return 5, 5 # file stage (4) plus two missing parents (2), minus one
+ return 1024 * 1024, 1024 * 1024
+
+ monkeypatch.setattr(secure_docker_environment, "_windows_artifact_disk_budget", disk_budget)
+
+ with pytest.raises(RuntimeError, match=r"insufficient disk space.*file publication"):
+ asyncio.run(environment.service_download_file("/remote/payload.bin", target))
+
+ assert not (tmp_path / "missing-one").exists()
+
+
+def test_windows_file_publication_enospc_preserves_existing_target(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_copy, secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ archive_buffer = io.BytesIO()
+ with tarfile.open(fileobj=archive_buffer, mode="w") as archive:
+ payload = b"replacement"
+ member = tarfile.TarInfo("./payload.bin")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ archive_payload = archive_buffer.getvalue()
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ original_copy = secure_copy.copy_file_secure
+
+ def copy_with_enospc(*args: object, **kwargs: object) -> None:
+ original_write = secure_copy.os.write
+ injected = False
+
+ def fail_after_partial_write(descriptor: int, data: bytes | memoryview) -> int:
+ nonlocal injected
+ written = original_write(descriptor, data)
+ if not injected:
+ injected = True
+ raise OSError(errno.ENOSPC, "injected publication disk exhaustion")
+ return written
+
+ with monkeypatch.context() as copy_patch:
+ copy_patch.setattr(secure_copy.os, "write", fail_after_partial_write)
+ original_copy(*args, **kwargs)
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ monkeypatch.setattr(secure_docker_environment, "copy_file_secure", copy_with_enospc)
+ target = tmp_path / "downloaded.bin"
+ target.write_bytes(b"old")
+
+ with pytest.raises(RuntimeError, match="could not be copied safely"):
+ asyncio.run(environment.service_download_file("/remote/payload.bin", target))
+
+ assert target.read_bytes() == b"old"
+ assert sorted(path.name for path in tmp_path.iterdir() if path.name.startswith(".downloaded.bin")) == []
+
+
+def test_windows_transfer_timeout_reaps_process_and_removes_partial_output(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "partial-transfer.tar"
+ exited = asyncio.Event()
+ signalled: list[signal.Signals] = []
+
+ class BlockedTransferProcess:
+ pid = 7645
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _FeedableStream()
+ self.stderr = _FeedableStream()
+
+ async def wait(self) -> int:
+ await exited.wait()
+ assert self.returncode is not None
+ return self.returncode
+
+ process = BlockedTransferProcess()
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> BlockedTransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+ process.stdout.feed_data(b"partial-untrusted-archive")
+ return process
+
+ def signal_process_tree(_process: object, value: signal.Signals) -> None:
+ signalled.append(value)
+ process.returncode = -int(value)
+ process.stdout.feed_eof()
+ process.stderr.feed_eof()
+ exited.set()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_signal_process_tree", signal_process_tree)
+
+ with pytest.raises(RuntimeError, match="secure Windows container transfer command timed out while idle"):
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ idle_timeout_sec=0.01,
+ )
+ )
+
+ assert signalled == [signal.SIGTERM]
+ assert not output_path.exists()
+
+
+def test_windows_transfer_idle_timeout_resets_while_output_progresses(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "progressing-transfer.tar"
+ exited = asyncio.Event()
+ producer: asyncio.Task[None] | None = None
+
+ class ProgressingTransferProcess:
+ pid = 7647
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _FeedableStream()
+ self.stderr = _FeedableStream()
+
+ async def wait(self) -> int:
+ await exited.wait()
+ assert self.returncode is not None
+ return self.returncode
+
+ process = ProgressingTransferProcess()
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> ProgressingTransferProcess:
+ nonlocal producer
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+
+ async def produce() -> None:
+ for index in range(8):
+ process.stdout.feed_data(bytes([index]))
+ await asyncio.sleep(0.01)
+ process.stdout.feed_eof()
+ process.stderr.feed_eof()
+ process.returncode = 0
+ exited.set()
+
+ producer = asyncio.create_task(produce())
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> None:
+ await environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ idle_timeout_sec=0.025,
+ )
+ assert producer is not None
+ await producer
+
+ asyncio.run(exercise())
+
+ assert output_path.read_bytes() == bytes(range(8))
+
+
+def test_windows_transfer_growing_output_exceeding_budget_reaps_process(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "growing-transfer.tar"
+ exited = asyncio.Event()
+ signalled: list[signal.Signals] = []
+ producer_tasks: list[asyncio.Task[None]] = []
+
+ class GrowingTransferProcess:
+ pid = 7651
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _FeedableStream()
+ self.stderr = _FeedableStream()
+
+ async def wait(self) -> int:
+ await exited.wait()
+ assert self.returncode is not None
+ return self.returncode
+
+ process = GrowingTransferProcess()
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> GrowingTransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+
+ async def produce() -> None:
+ while not exited.is_set():
+ process.stdout.feed_data(b"grow")
+ await asyncio.sleep(0.002)
+
+ producer_tasks.append(asyncio.create_task(produce()))
+ return process
+
+ def signal_process_tree(_process: object, value: signal.Signals) -> None:
+ signalled.append(value)
+ process.returncode = -int(value)
+ process.stdout.feed_eof()
+ process.stderr.feed_eof()
+ exited.set()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_signal_process_tree", signal_process_tree)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TRANSFER_POLL_SECONDS", 0.001)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_DISK_RESERVE_BYTES", 0)
+ monkeypatch.setattr(
+ secure_docker_environment.shutil,
+ "disk_usage",
+ lambda _path: SimpleNamespace(free=10),
+ )
+
+ with pytest.raises(RuntimeError, match="exceeded its temporary disk budget"):
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ )
+ )
+
+ assert signalled == [signal.SIGTERM]
+ assert all(task.done() for task in producer_tasks)
+ assert not output_path.exists()
+
+
+def test_windows_transfer_total_timeout_reaps_continuously_progressing_process(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "trickling-transfer.tar"
+ exited = asyncio.Event()
+ signalled: list[signal.Signals] = []
+ producer_tasks: list[asyncio.Task[None]] = []
+
+ class TricklingTransferProcess:
+ pid = 7652
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _FeedableStream()
+ self.stderr = _FeedableStream()
+
+ async def wait(self) -> int:
+ await exited.wait()
+ assert self.returncode is not None
+ return self.returncode
+
+ process = TricklingTransferProcess()
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> TricklingTransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+
+ async def produce() -> None:
+ while not exited.is_set():
+ process.stdout.feed_data(b"x")
+ await asyncio.sleep(0.002)
+
+ producer_tasks.append(asyncio.create_task(produce()))
+ return process
+
+ def signal_process_tree(_process: object, value: signal.Signals) -> None:
+ signalled.append(value)
+ process.returncode = -int(value)
+ process.stdout.feed_eof()
+ process.stderr.feed_eof()
+ exited.set()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_signal_process_tree", signal_process_tree)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TRANSFER_POLL_SECONDS", 0.001)
+
+ with pytest.raises(RuntimeError, match="exceeded its total time limit"):
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ idle_timeout_sec=0.02,
+ total_timeout_sec=0.01,
+ )
+ )
+
+ assert signalled == [signal.SIGTERM]
+ assert all(task.done() for task in producer_tasks)
+ assert not output_path.exists()
+
+
+@pytest.mark.parametrize(
+ ("timeout_name", "timeout_value"),
+ (
+ ("idle_timeout_sec", 0.0),
+ ("idle_timeout_sec", -1.0),
+ ("idle_timeout_sec", float("nan")),
+ ("idle_timeout_sec", float("inf")),
+ ("total_timeout_sec", 0.0),
+ ("total_timeout_sec", -1.0),
+ ("total_timeout_sec", float("nan")),
+ ("total_timeout_sec", float("inf")),
+ ),
+)
+def test_windows_transfer_rejects_nonpositive_or_nonfinite_timeouts(
+ tmp_path: Path,
+ timeout_name: str,
+ timeout_value: float,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+
+ with pytest.raises(ValueError, match="must be a positive finite value"):
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=tmp_path / "invalid-timeout.tar",
+ **{timeout_name: timeout_value},
+ )
+ )
+
+
+def test_windows_transfer_rejects_process_completion_after_total_deadline(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "late-completion.tar"
+ signalled: list[signal.Signals] = []
+
+ class LateTransferProcess:
+ pid = 7653
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _ChunkStream([])
+ self.stderr = _ChunkStream([])
+
+ async def wait(self) -> int:
+ await asyncio.sleep(0.02)
+ self.returncode = 0
+ return self.returncode
+
+ process = LateTransferProcess()
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> LateTransferProcess:
+ return process
+
+ def signal_process_tree(_process: object, value: signal.Signals) -> None:
+ signalled.append(value)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_signal_process_tree", signal_process_tree)
+
+ with pytest.raises(RuntimeError, match="exceeded its total time limit"):
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ total_timeout_sec=0.01,
+ )
+ )
+
+ assert signalled == [signal.SIGTERM]
+ assert not output_path.exists()
+
+
+def test_windows_transfer_repeated_cancellation_reaps_process_and_removes_partial_output(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "cancelled-transfer.tar"
+ started = asyncio.Event()
+ exited = asyncio.Event()
+ signalled: list[signal.Signals] = []
+
+ class BlockedTransferProcess:
+ pid = 7646
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _FeedableStream()
+ self.stderr = _FeedableStream()
+
+ async def wait(self) -> int:
+ await exited.wait()
+ assert self.returncode is not None
+ return self.returncode
+
+ process = BlockedTransferProcess()
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> BlockedTransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+ process.stdout.feed_data(b"partial-cancelled-archive")
+ started.set()
+ return process
+
+ def signal_process_tree(_process: object, value: signal.Signals) -> None:
+ signalled.append(value)
+ process.returncode = -int(value)
+ process.stdout.feed_eof()
+ process.stderr.feed_eof()
+ exited.set()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_signal_process_tree", signal_process_tree)
+
+ async def exercise() -> None:
+ transfer = asyncio.create_task(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values=set(),
+ output_path=output_path,
+ )
+ )
+ await started.wait()
+ transfer.cancel()
+ await asyncio.sleep(0)
+ transfer.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await transfer
+
+ asyncio.run(exercise())
+
+ assert signalled == [signal.SIGTERM]
+ assert not output_path.exists()
+
+
+@pytest.mark.parametrize(
+ ("member_type", "member_name", "link_name"),
+ (
+ (tarfile.SYMTYPE, "./payload.bin", "/host/outside.txt"),
+ (tarfile.LNKTYPE, "./payload.bin", "./other.bin"),
+ (tarfile.FIFOTYPE, "./payload.bin", ""),
+ (tarfile.REGTYPE, "../payload.bin", ""),
+ (tarfile.REGTYPE, "/payload.bin", ""),
+ ),
+)
+def test_windows_main_download_rejects_unsafe_container_archive_members(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ member_type: bytes,
+ member_name: str,
+ link_name: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ outside = tmp_path / "outside.txt"
+ outside.write_text("host-only-content", encoding="utf-8")
+
+ archive_buffer = io.BytesIO()
+ with tarfile.open(fileobj=archive_buffer, mode="w") as archive:
+ member = tarfile.TarInfo(member_name)
+ member.type = member_type
+ member.linkname = str(outside) if member_type == tarfile.SYMTYPE else link_name
+ if member_type == tarfile.REGTYPE:
+ payload = b"container-controlled-content"
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ else:
+ archive.addfile(member)
+ archive_payload = archive_buffer.getvalue()
+
+ async def transfer(command: list[str], **kwargs: object) -> None:
+ assert command[0] == "docker"
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ downloaded = tmp_path / "downloaded.bin"
+
+ with pytest.raises(RuntimeError, match="unsafe Windows container download archive"):
+ asyncio.run(environment.service_download_file("/remote/payload.bin", downloaded))
+
+ assert not downloaded.exists()
+ assert outside.read_text(encoding="utf-8") == "host-only-content"
+
+
+def test_windows_tar_member_rejects_many_leading_current_components_before_filter(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ class NoFrontPopParts(list[str]):
+ def pop(self, index: int = -1) -> str:
+ if index == 0:
+ raise AssertionError("front-pop path normalization is quadratic")
+ return super().pop(index)
+
+ class ManyLeadingCurrentComponents(str):
+ __slots__ = ()
+
+ def split(self, *args: object, **kwargs: object) -> NoFrontPopParts:
+ return NoFrontPopParts(super().split(*args, **kwargs))
+
+ member = tarfile.TarInfo(ManyLeadingCurrentComponents("./" * 100_000 + "payload.bin"))
+ monkeypatch.setattr(
+ tarfile,
+ "data_filter",
+ lambda _candidate, _target: pytest.fail("unbounded path reached tarfile filter"),
+ )
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._validated_windows_tar_member(member, tmp_path)
+
+
+def test_windows_tar_member_accepts_bounded_leading_current_components(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ member = tarfile.TarInfo(
+ "./" * secure_docker_environment._WINDOWS_TAR_MAX_LEADING_CURRENT_COMPONENTS + "payload.bin"
+ )
+ monkeypatch.setattr(tarfile, "data_filter", lambda candidate, _target: candidate)
+
+ validated = secure_docker_environment._validated_windows_tar_member(member, tmp_path)
+
+ assert validated is not None
+ assert validated.canonical_parts == ("payload.bin",)
+
+
+@pytest.mark.parametrize(
+ "member_names",
+ (
+ ("./payload.bin", "./payload.bin"),
+ ("./Readme", "./README"),
+ ("./café", "./cafe\N{COMBINING ACUTE ACCENT}"),
+ ("./file", "./file/child"),
+ ("./file/child", "./file"),
+ ("./name:alternate-stream",),
+ ("./CON.txt",),
+ ("./CONIN$",),
+ ("./CONOUT$.log",),
+ ("./COM1",),
+ ("./COM9.log",),
+ ("./LPT1",),
+ ("./LPT9.txt",),
+ ("./COM¹",),
+ ("./LPT³.txt",),
+ ("./trailing.",),
+ ("./trailing-space ",),
+ ),
+)
+def test_windows_main_download_rejects_aliases_reserved_names_and_file_prefixes(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ member_names: tuple[str, ...],
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ archive_buffer = io.BytesIO()
+ with tarfile.open(fileobj=archive_buffer, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ for index, member_name in enumerate(member_names):
+ payload = f"payload-{index}".encode()
+ member = tarfile.TarInfo(member_name)
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ archive_payload = archive_buffer.getvalue()
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ target = tmp_path / "existing-target"
+ target.mkdir()
+ sentinel = target / "sentinel.bin"
+ sentinel.write_bytes(b"unchanged")
+
+ with pytest.raises(RuntimeError, match="unsafe Windows container download archive"):
+ asyncio.run(environment.service_download_dir("/remote/tree", target))
+
+ assert list(target.iterdir()) == [sentinel]
+ assert sentinel.read_bytes() == b"unchanged"
+
+
+@pytest.mark.parametrize("member_name", ("./COM0", "./COM0.txt", "./LPT0", "./LPT0.log"))
+def test_windows_tar_member_accepts_nonreserved_zero_suffixed_device_names(
+ tmp_path: Path,
+ member_name: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ member = tarfile.TarInfo(member_name)
+
+ validated = secure_docker_environment._validated_windows_tar_member(member, tmp_path)
+
+ assert validated is not None
+ assert validated.kind == "file"
+
+
+@pytest.mark.parametrize("archive_kind", ["truncated", "compressed"])
+def test_windows_main_download_rejects_malformed_or_compressed_archive_before_target_mutation(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ archive_kind: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ archive_buffer = io.BytesIO()
+ mode = "w:gz" if archive_kind == "compressed" else "w"
+ with tarfile.open(fileobj=archive_buffer, mode=mode) as archive:
+ payload = b"container-content"
+ member = tarfile.TarInfo("./payload.bin")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ archive_payload = archive_buffer.getvalue()
+ if archive_kind == "truncated":
+ archive_payload = archive_payload[:700]
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ target = tmp_path / "existing-target"
+ target.mkdir()
+ sentinel = target / "sentinel.bin"
+ sentinel.write_bytes(b"unchanged")
+
+ with pytest.raises(RuntimeError, match="unsafe Windows container download archive"):
+ asyncio.run(environment.service_download_dir("/remote/tree", target))
+
+ assert list(target.iterdir()) == [sentinel]
+ assert sentinel.read_bytes() == b"unchanged"
+
+
+@pytest.mark.parametrize("bound", ["members", "paths", "components", "metadata", "disk"])
+def test_windows_main_download_enforces_archive_resource_bounds_before_target_mutation(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ bound: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ archive_buffer = io.BytesIO()
+ long_name = "./" + "long-name-" * 20 + "payload.bin"
+ with tarfile.open(fileobj=archive_buffer, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ names = (
+ ("./first.bin", "./second.bin")
+ if bound == "members"
+ else (
+ "./nested/payload.bin"
+ if bound == "components"
+ else long_name
+ if bound in {"paths", "metadata"}
+ else "./payload.bin",
+ )
+ )
+ for name in names:
+ payload = b"four"
+ member = tarfile.TarInfo(name)
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ archive_payload = archive_buffer.getvalue()
+
+ if bound == "members":
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TAR_MAX_MEMBERS", 1)
+ elif bound == "paths":
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TAR_MAX_PATH_BYTES", 8)
+ elif bound == "components":
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TAR_MAX_PATH_COMPONENTS", 1)
+ elif bound == "metadata":
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TAR_MAX_EXTENSION_BYTES", 8)
+ else:
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_DISK_RESERVE_BYTES", 0)
+ monkeypatch.setattr(
+ secure_docker_environment.shutil,
+ "disk_usage",
+ lambda _path: SimpleNamespace(free=3),
+ )
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ target = tmp_path / "existing-target"
+ target.mkdir()
+ sentinel = target / "sentinel.bin"
+ sentinel.write_bytes(b"unchanged")
+
+ expected_error = (
+ "insufficient disk space.*archive extraction"
+ if bound == "disk"
+ else "unsafe Windows container download archive"
+ )
+ with pytest.raises(RuntimeError, match=expected_error):
+ asyncio.run(environment.service_download_dir("/remote/tree", target))
+
+ assert list(target.iterdir()) == [sentinel]
+ assert sentinel.read_bytes() == b"unchanged"
+
+
+@pytest.mark.parametrize(
+ "extension_type",
+ (
+ tarfile.XHDTYPE,
+ tarfile.XGLTYPE,
+ tarfile.SOLARIS_XHDTYPE,
+ tarfile.GNUTYPE_LONGNAME,
+ tarfile.GNUTYPE_LONGLINK,
+ ),
+)
+def test_windows_tar_prescan_bounds_every_allocating_extension_type(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ extension_type: bytes,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "metadata-extension.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ payload = b"1 path=payload.bin\n" + b"x" * 64
+ member = tarfile.TarInfo("pax-metadata")
+ member.type = extension_type
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TAR_MAX_EXTENSION_BYTES", 8)
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_rejects_global_pax_even_below_metadata_limit(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "global-pax.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ member = tarfile.TarInfo("global-pax")
+ member.type = tarfile.XGLTYPE
+ member.size = 1
+ archive.addfile(member, io.BytesIO(b"x"))
+ for index in range(32):
+ payload = b""
+ regular = tarfile.TarInfo(f"payload-{index}.bin")
+ regular.size = len(payload)
+ archive.addfile(regular, io.BytesIO(payload))
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+@pytest.mark.parametrize(
+ "sparse_keyword",
+ (
+ "GNU.sparse.map",
+ "GNU.sparse.size",
+ "GNU.sparse.name",
+ "GNU.sparse.realsize",
+ "GNU.sparse.major",
+ "GNU.sparse.minor",
+ ),
+)
+def test_windows_tar_prescan_rejects_local_pax_gnu_sparse_metadata(
+ tmp_path: Path,
+ sparse_keyword: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "gnu-sparse-pax.tar"
+ with tarfile.open(archive_path, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ payload = b"1\n0\n0\n"
+ member = tarfile.TarInfo("payload.bin")
+ member.pax_headers = {sparse_keyword: "1"}
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_rejects_gnu_sparse_v1_before_payload_parsing(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "gnu-sparse-v1.tar"
+ with tarfile.open(archive_path, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ payload = (b"0\n0\n" * 100_000) + b"0\n"
+ member = tarfile.TarInfo("payload.bin")
+ member.pax_headers = {
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ }
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_rejects_pax_size_tunneling_hidden_sparse_metadata(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ hidden_archive = io.BytesIO()
+ with tarfile.open(fileobj=hidden_archive, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ sparse_map = (b"0\n0\n" * 100_000) + b"0\n"
+ member = tarfile.TarInfo("hidden-sparse.bin")
+ member.pax_headers = {
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ }
+ member.size = len(sparse_map)
+ archive.addfile(member, io.BytesIO(sparse_map))
+
+ archive_path = tmp_path / "pax-size-tunnel.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ size_override = b"10 size=0\n"
+ pax = tarfile.TarInfo("size-override")
+ pax.type = tarfile.XHDTYPE
+ pax.size = len(size_override)
+ archive.addfile(pax, io.BytesIO(size_override))
+
+ hidden_payload = hidden_archive.getvalue()
+ carrier = tarfile.TarInfo("carrier.bin")
+ carrier.size = len(hidden_payload)
+ archive.addfile(carrier, io.BytesIO(hidden_payload))
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_rejects_pax_size_tunneling_hidden_global_pax(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ hidden_archive = io.BytesIO()
+ with tarfile.open(fileobj=hidden_archive, mode="w") as archive:
+ global_pax_payload = b"18 comment=hidden\n"
+ global_pax = tarfile.TarInfo("hidden-global-pax")
+ global_pax.type = tarfile.XGLTYPE
+ global_pax.size = len(global_pax_payload)
+ archive.addfile(global_pax, io.BytesIO(global_pax_payload))
+ regular = tarfile.TarInfo("payload.bin")
+ regular.size = 0
+ archive.addfile(regular, io.BytesIO())
+
+ archive_path = tmp_path / "pax-size-global-tunnel.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ size_override = b"10 size=0\n"
+ pax = tarfile.TarInfo("size-override")
+ pax.type = tarfile.XHDTYPE
+ pax.size = len(size_override)
+ archive.addfile(pax, io.BytesIO(size_override))
+
+ hidden_payload = hidden_archive.getvalue()
+ carrier = tarfile.TarInfo("carrier.bin")
+ carrier.size = len(hidden_payload)
+ archive.addfile(carrier, io.BytesIO(hidden_payload))
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_bounds_local_pax_record_count(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "many-pax-records.tar"
+ with tarfile.open(archive_path, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ member = tarfile.TarInfo("payload.bin")
+ member.pax_headers = {"comment": "one", "atime": "two"}
+ member.size = 0
+ archive.addfile(member, io.BytesIO())
+
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "_WINDOWS_TAR_MAX_PAX_RECORDS_PER_HEADER",
+ 1,
+ )
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_resets_pax_record_count_between_headers(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "separate-pax-records.tar"
+ with tarfile.open(archive_path, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ for index in range(2):
+ member = tarfile.TarInfo(f"payload-{index}.bin")
+ member.pax_headers = {"comment": str(index)}
+ member.size = 0
+ archive.addfile(member, io.BytesIO())
+
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "_WINDOWS_TAR_MAX_PAX_RECORDS_PER_HEADER",
+ 1,
+ )
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_accepts_bounded_ordinary_local_pax_metadata(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "ordinary-pax.tar"
+ with tarfile.open(archive_path, mode="w", format=tarfile.PAX_FORMAT) as archive:
+ payload = b"content"
+ member = tarfile.TarInfo("payload.bin")
+ member.pax_headers = {"comment": "ordinary metadata"}
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_follows_bounded_local_pax_size_override(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ payload = b"content"
+ size_override = b"9 size=7\n"
+ pax = tarfile.TarInfo("size-override")
+ pax.type = tarfile.XHDTYPE
+ pax.size = len(size_override)
+ carrier = tarfile.TarInfo("payload.bin")
+ carrier.size = 0
+ archive_bytes = (
+ pax.tobuf(format=tarfile.USTAR_FORMAT)
+ + size_override.ljust(512, b"\0")
+ + carrier.tobuf(format=tarfile.USTAR_FORMAT)
+ + payload.ljust(512, b"\0")
+ + b"\0" * 1024
+ )
+ archive_path = tmp_path / "pax-size-override.tar"
+ archive_path.write_bytes(archive_bytes)
+
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+ with tarfile.open(archive_path, mode="r:") as archive:
+ member = archive.next()
+ assert member is not None
+ assert member.size == len(payload)
+ extracted = archive.extractfile(member)
+ assert extracted is not None
+ assert extracted.read() == payload
+
+
+def test_windows_tar_prescan_bounds_zero_byte_extension_chains(
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "extension-chain.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ for index in range(secure_docker_environment._WINDOWS_TAR_MAX_EXTENSION_CHAIN + 1):
+ extension = tarfile.TarInfo(f"pax-{index}")
+ extension.type = tarfile.XHDTYPE
+ extension.size = 0
+ archive.addfile(extension, io.BytesIO())
+ regular = tarfile.TarInfo("payload.bin")
+ regular.size = 0
+ archive.addfile(regular, io.BytesIO())
+
+ with pytest.raises(ValueError):
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_tar_prescan_resets_extension_chain_after_regular_members(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "separated-extensions.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ for index in range(2):
+ extension = tarfile.TarInfo(f"pax-{index}")
+ extension.type = tarfile.XHDTYPE
+ extension.size = 0
+ archive.addfile(extension, io.BytesIO())
+ regular = tarfile.TarInfo(f"payload-{index}.bin")
+ regular.size = 0
+ archive.addfile(regular, io.BytesIO())
+
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TAR_MAX_EXTENSION_CHAIN", 1)
+ secure_docker_environment._prescan_uncompressed_tar_archive(archive_path)
+
+
+def test_windows_archive_budget_counts_zero_byte_entries_before_target_mutation(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+ environment._windows_container_name = "harbor-secure-windows"
+ archive_buffer = io.BytesIO()
+ with tarfile.open(fileobj=archive_buffer, mode="w") as archive:
+ for index in range(3):
+ member = tarfile.TarInfo(f"./empty-{index}.bin")
+ member.size = 0
+ archive.addfile(member, io.BytesIO())
+ archive_payload = archive_buffer.getvalue()
+
+ async def transfer(_command: list[str], **kwargs: object) -> None:
+ Path(kwargs["output_path"]).write_bytes(archive_payload) # type: ignore[arg-type]
+
+ monkeypatch.setattr(environment, "_run_trusted_transfer_command", transfer)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_ENTRY_DISK_BYTES", 10)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_DISK_RESERVE_BYTES", 0)
+ monkeypatch.setattr(
+ secure_docker_environment.shutil,
+ "disk_usage",
+ lambda _path: SimpleNamespace(free=39),
+ )
+ target = tmp_path / "existing-target"
+ target.mkdir()
+ sentinel = target / "sentinel.bin"
+ sentinel.write_bytes(b"unchanged")
+
+ with pytest.raises(RuntimeError, match=r"insufficient disk space.*archive extraction"):
+ asyncio.run(environment.service_download_dir("/remote/tree", target))
+
+ assert list(target.iterdir()) == [sentinel]
+ assert sentinel.read_bytes() == b"unchanged"
+
+
+@pytest.mark.parametrize(("available_bytes", "should_succeed"), ((34, True), (33, False)))
+def test_windows_archive_extraction_enforces_exact_estimated_byte_boundary(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ available_bytes: int,
+ should_succeed: bool,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "boundary.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ payload = b"four"
+ member = tarfile.TarInfo("./nested/payload.bin")
+ member.size = len(payload)
+ archive.addfile(member, io.BytesIO(payload))
+ target = tmp_path / "extracted"
+ target.mkdir()
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_ENTRY_DISK_BYTES", 10)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_DISK_RESERVE_BYTES", 0)
+ monkeypatch.setattr(
+ secure_docker_environment.shutil,
+ "disk_usage",
+ lambda _path: SimpleNamespace(free=available_bytes),
+ )
+
+ if should_succeed:
+ usage = secure_docker_environment._extract_regular_tar_archive(archive_path, target)
+ assert usage.estimated_disk_bytes == available_bytes
+ assert (target / "nested" / "payload.bin").read_bytes() == b"four"
+ else:
+ with pytest.raises(RuntimeError, match="insufficient disk space"):
+ secure_docker_environment._extract_regular_tar_archive(archive_path, target)
+ assert list(target.iterdir()) == []
+
+
+@pytest.mark.parametrize("resource", ["entries", "inodes"])
+def test_windows_archive_rejects_implicit_directory_resource_exhaustion(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ resource: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ archive_path = tmp_path / "implicit-directories.tar"
+ with tarfile.open(archive_path, mode="w") as archive:
+ member = tarfile.TarInfo("./one/two/payload.bin")
+ member.size = 0
+ archive.addfile(member, io.BytesIO())
+ target = tmp_path / "extracted"
+ target.mkdir()
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_ARTIFACT_DISK_RESERVE_BYTES", 0)
+ if resource == "entries":
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TAR_MAX_FILESYSTEM_ENTRIES", 3)
+ expected_error = "unsafe Windows container download archive"
+ else:
+ monkeypatch.setattr(secure_docker_environment, "_windows_artifact_inode_budget", lambda _path: 3)
+ expected_error = "insufficient filesystem entries"
+
+ with pytest.raises(RuntimeError, match=expected_error):
+ secure_docker_environment._extract_regular_tar_archive(archive_path, target)
+
+ assert list(target.iterdir()) == []
+
+
+def test_windows_transfer_error_redacts_exact_short_sensitive_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ secret = "x"
+ output_path = tmp_path / "failed-transfer.tar"
+
+ class FailedTransferProcess:
+ pid = 7644
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _ChunkStream([])
+ self.stderr = _ChunkStream([("discarded-prefix-" * 20 + f"|{secret}|transfer failed").encode()])
+
+ async def wait(self) -> int:
+ self.returncode = 9
+ return self.returncode
+
+ async def create_subprocess(
+ *_args: object,
+ **kwargs: object,
+ ) -> FailedTransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+ return FailedTransferProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TRANSFER_STDERR_MAX_BYTES", 64)
+
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values={secret},
+ output_path=output_path,
+ )
+ )
+
+ assert secret not in str(caught.value)
+ assert "diagnostics omitted" in str(caught.value)
+ assert len(str(caught.value)) < 256
+ assert not output_path.exists()
+
+
+def test_windows_transfer_truncated_error_cannot_expose_secret_suffix(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ secret = "SYNTHETIC_SECRET_ABCDEF"
+ output_path = tmp_path / "failed-transfer.tar"
+
+ class FailedTransferProcess:
+ pid = 7649
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _ChunkStream([])
+ self.stderr = _ChunkStream([f"prefix|{secret}|FAIL".encode()])
+
+ async def wait(self) -> int:
+ self.returncode = 9
+ return self.returncode
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> FailedTransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+ return FailedTransferProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TRANSFER_STDERR_MAX_BYTES", 16)
+
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values={secret},
+ output_path=output_path,
+ )
+ )
+
+ rendered = str(caught.value)
+ assert secret not in rendered
+ assert "CRET_ABCDEF" not in rendered
+ assert "diagnostics omitted" in rendered
+ assert not output_path.exists()
+
+
+def test_windows_transfer_truncated_error_with_repeated_secret_cannot_expose_earlier_suffix(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ secret = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567"
+ leaked_suffix = "RSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567"
+ output_path = tmp_path / "failed-transfer.tar"
+
+ class FailedTransferProcess:
+ pid = 7654
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _ChunkStream([])
+ self.stderr = _ChunkStream([("X" + secret + "|" + secret + "!" * 8).encode()])
+
+ async def wait(self) -> int:
+ self.returncode = 9
+ return self.returncode
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> FailedTransferProcess:
+ return FailedTransferProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment, "_WINDOWS_TRANSFER_STDERR_MAX_BYTES", 64)
+
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={"PATH": os.environ["PATH"]},
+ protected_values={secret},
+ output_path=output_path,
+ )
+ )
+
+ rendered = str(caught.value)
+ assert secret not in rendered
+ assert leaked_suffix not in rendered
+ assert "diagnostics omitted" in rendered
+ assert not output_path.exists()
+
+
+@pytest.mark.parametrize(
+ ("environment_name", "secret"),
+ (
+ ("DOCKER_AUTH_CONFIG", "SYNTHETIC_DOCKER_AUTH_SECRET"),
+ ("DOCKER_HOST", "tcp://synthetic-user:synthetic-pass@example.invalid:2376"),
+ ("DOCKER_HOST", "tcp://synthetic-user:synthetic-pass@[malformed"),
+ ("HTTPS_PROXY", "synthetic-user:synthetic-pass@example.invalid:8080"),
+ ),
+)
+def test_windows_transfer_error_redacts_sensitive_docker_client_environment(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ environment_name: str,
+ secret: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ output_path = tmp_path / "failed-transfer.tar"
+
+ class FailedTransferProcess:
+ pid = 7650
+
+ def __init__(self) -> None:
+ self.returncode: int | None = None
+ self.stdout = _ChunkStream([])
+ self.stderr = _ChunkStream([f"docker auth failed: {secret}".encode()])
+
+ async def wait(self) -> int:
+ self.returncode = 9
+ return self.returncode
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> FailedTransferProcess:
+ assert kwargs["stdout"] == asyncio.subprocess.PIPE
+ assert kwargs["stderr"] == asyncio.subprocess.PIPE
+ return FailedTransferProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(
+ environment._run_trusted_transfer_command(
+ ["docker", "version"],
+ process_environment={
+ "PATH": os.environ["PATH"],
+ environment_name: secret,
+ },
+ protected_values=set(),
+ output_path=output_path,
+ )
+ )
+
+ assert secret not in str(caught.value)
+ assert _collision_safe_redaction_marker({secret}, include_short=True) in str(caught.value)
+ assert not output_path.exists()
+
+
+@pytest.mark.parametrize("operation", ["file", "dir"])
+def test_service_download_rejects_empty_service_instead_of_crossing_into_main(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ operation: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ spawned = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ nonlocal spawned
+ spawned = True
+ return _BufferedComposeProcess()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(ValueError, match="Invalid Docker Compose service name"):
+ if operation == "file":
+ asyncio.run(
+ environment.service_download_file(
+ "/tmp/source",
+ tmp_path / "target",
+ service="",
+ )
+ )
+ else:
+ asyncio.run(
+ environment.service_download_dir(
+ "/tmp/source",
+ tmp_path / "target",
+ service="",
+ )
+ )
+
+ assert spawned is False
+
+
+def test_sidecar_service_exec_rejects_windows_with_public_harbor_error(tmp_path: Path) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ environment._is_windows_container = True
+
+ with pytest.raises(ServiceOperationsUnsupportedError, match="requested service: 'helper'"):
+ asyncio.run(environment.service_exec("echo unsupported", service="helper"))
+
+
+def test_sidecar_callback_base_exception_reaps_client_without_stopping_main(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ secret = "sidecar-callback-base-error-secret"
+ callback_text: list[str] = []
+ main_containment_calls = 0
+ sidecar_containment_calls: list[str] = []
+ child_processes: list[asyncio.subprocess.Process] = []
+ real_create_subprocess = asyncio.create_subprocess_exec
+
+ async def create_host_subprocess(*_args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await real_create_subprocess(
+ sys.executable,
+ "-c",
+ f"print({secret!r})",
+ **kwargs,
+ )
+ child_processes.append(process)
+ return process
+
+ async def contain_main() -> None:
+ nonlocal main_containment_calls
+ main_containment_calls += 1
+
+ async def contain_sidecar(service: str) -> None:
+ sidecar_containment_calls.append(service)
+
+ callback_error = _CallbackBaseError("sidecar callback failed")
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_text.append(text)
+ raise callback_error
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_host_subprocess)
+ monkeypatch.setattr(environment, "_contain_main_container", contain_main)
+ monkeypatch.setattr(environment, "_contain_sidecar_service", contain_sidecar, raising=False)
+
+ async def exercise() -> None:
+ with environment.scoped_output_callback(on_output):
+ with pytest.raises(_CallbackBaseError, match="sidecar callback failed") as caught:
+ await environment.service_exec(
+ "emit-sidecar-secret",
+ service="helper",
+ env={"SIDECAR_TOKEN": secret},
+ )
+ assert caught.value is callback_error
+
+ asyncio.run(exercise())
+
+ assert callback_text == [f"{_marker_for(secret)}\n"]
+ assert main_containment_calls == 0
+ assert sidecar_containment_calls == ["helper"]
+ assert len(child_processes) == 1
+ assert child_processes[0].returncode is not None
+
+
+@pytest.mark.parametrize("reap_fails", [False, True])
+def test_sidecar_containment_reaps_expired_client_before_restoration(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ reap_fails: bool,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ actions: list[str] = []
+
+ contained_snapshot = secure_docker_environment._RawServiceSnapshot(
+ all_identities=("1" * 64,),
+ running_identities=("1" * 64,),
+ )
+
+ async def contain(service: str) -> object:
+ assert service == "helper"
+ actions.append("contained")
+ return contained_snapshot
+
+ async def reap(
+ _process: object,
+ _communication: asyncio.Task[object],
+ *,
+ preserve_cancellation: bool,
+ ) -> None:
+ assert preserve_cancellation is False
+ actions.append("reaped")
+ if reap_fails:
+ raise PermissionError("host client reap denied")
+
+ async def restore(service: str, *, snapshot: object) -> bool:
+ assert service == "helper"
+ assert snapshot == contained_snapshot
+ assert actions == ["contained", "reaped"]
+ actions.append("restored")
+ return True
+
+ monkeypatch.setattr(environment, "_contain_sidecar_service", contain)
+ monkeypatch.setattr(environment, "_restore_sidecar_service", restore)
+ monkeypatch.setattr(secure_docker_environment, "_terminate_process_tree", reap)
+
+ async def exercise() -> None:
+ communication = asyncio.create_task(asyncio.sleep(0))
+ if reap_fails:
+ with pytest.raises(RuntimeError, match="containment and restoration") as caught:
+ await environment._contain_main_and_reap_compose(
+ SimpleNamespace(), # type: ignore[arg-type]
+ communication,
+ contain_service_on_interrupt="helper",
+ stop_main_on_interrupt=False,
+ )
+ assert isinstance(caught.value.__cause__, PermissionError)
+ else:
+ await environment._contain_main_and_reap_compose(
+ SimpleNamespace(), # type: ignore[arg-type]
+ communication,
+ contain_service_on_interrupt="helper",
+ stop_main_on_interrupt=False,
+ )
+
+ asyncio.run(exercise())
+
+ assert actions == (["contained", "reaped"] if reap_fails else ["contained", "reaped", "restored"])
+
+
+@pytest.mark.parametrize("interrupt_mode", ["timeout", "cancel"])
+def test_sidecar_interrupt_reaps_client_without_stopping_main(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ interrupt_mode: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.05)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.05)
+ environment = _initialized_secure_docker_environment(tmp_path)
+ main_containment_calls = 0
+ sidecar_containment_calls: list[str] = []
+ child_processes: list[asyncio.subprocess.Process] = []
+ child_created = asyncio.Event()
+ real_create_subprocess = asyncio.create_subprocess_exec
+
+ async def create_host_subprocess(*_args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ process = await real_create_subprocess(
+ sys.executable,
+ "-c",
+ "import time; time.sleep(30)",
+ **kwargs,
+ )
+ child_processes.append(process)
+ child_created.set()
+ return process
+
+ async def contain_main() -> None:
+ nonlocal main_containment_calls
+ main_containment_calls += 1
+
+ async def contain_sidecar(service: str) -> None:
+ sidecar_containment_calls.append(service)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_host_subprocess)
+ monkeypatch.setattr(environment, "_contain_main_container", contain_main)
+ monkeypatch.setattr(environment, "_contain_sidecar_service", contain_sidecar, raising=False)
+
+ async def exercise() -> None:
+ task = asyncio.create_task(
+ environment.service_exec(
+ "sleep 30",
+ service="helper",
+ timeout_sec=0.02 if interrupt_mode == "timeout" else None,
+ )
+ )
+ await asyncio.wait_for(child_created.wait(), timeout=1)
+ if interrupt_mode == "timeout":
+ with pytest.raises(RuntimeError, match="timed out"):
+ await task
+ else:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ asyncio.run(exercise())
+
+ assert main_containment_calls == 0
+ assert sidecar_containment_calls == ["helper"]
+ assert len(child_processes) == 1
+ assert child_processes[0].returncode is not None
+
+
+def test_upload_file_compose_cp_failure_forwards_exact_tar_bytes_to_stdin(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_docker_environment(tmp_path)
+ source = tmp_path / "binary-payload.bin"
+ source.write_bytes(b"\x00tar-fallback\xff\nwith spaces\x00")
+ expected_tar = environment._platform._tar_file(source, "uploaded-payload.bin")
+ calls: list[tuple[list[str], bool, bytes | None]] = []
+
+ async def run_compose(
+ command: list[str],
+ check: bool = True,
+ timeout_sec: float | None = None,
+ stdin_data: bytes | None = None,
+ on_output: object | None = None,
+ **_kwargs: object,
+ ) -> ExecResult:
+ del timeout_sec, on_output
+ calls.append((command, check, stdin_data))
+ if command[0] == "cp":
+ raise RuntimeError("compose cp deliberately unavailable")
+ if "test" in command:
+ return ExecResult(stdout=None, stderr=None, return_code=1)
+ return ExecResult(stdout=None, stderr=None, return_code=0)
+
+ monkeypatch.setattr(environment, "_run_docker_compose_command", run_compose)
+
+ asyncio.run(environment.upload_file(source, "/tmp/uploaded-payload.bin"))
+
+ assert calls[0][0][0] == "cp"
+ tar_calls = [call for call in calls if "tar" in call[0]]
+ assert len(tar_calls) == 1
+ assert tar_calls[0][0] == [
+ "exec",
+ "-T",
+ "-u",
+ "root",
+ MAIN_SERVICE_NAME,
+ "tar",
+ "-xf",
+ "-",
+ "-C",
+ "/tmp",
+ ]
+ assert tar_calls[0][1] is True
+ assert tar_calls[0][2] == expected_tar
+
+
+def test_secure_public_exec_without_environment_still_streams(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ process = _BufferedAndStreamedComposeProcess([b"plain output\n"])
+ callback_chunks: list[str] = []
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedAndStreamedComposeProcess:
+ return process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec("emit-plain-output")
+
+ result = asyncio.run(exercise())
+
+ assert "".join(callback_chunks) == result.stdout == "plain output\n"
+ assert process.communicate_inputs == []
+
+
+def test_secure_public_exec_streams_redacted_handoff_secrets(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persistent_secret = "secure-persistent-callback-secret"
+ per_call_secret = "secure-per-call-callback-secret"
+ scoped_secret = "secure-scoped-callback-secret"
+ secrets = {persistent_secret, per_call_secret, scoped_secret}
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"PERSISTENT_TOKEN": persistent_secret},
+ )
+ callback_events: list[tuple[str, str, str]] = []
+ subprocess_commands: list[tuple[object, ...]] = []
+ handoff_process = _BufferedComposeProcess(stdout=b"", return_code=0)
+ main_process = _BufferedAndStreamedComposeProcess(
+ [
+ f"persistent {persistent_secret}\n".encode(),
+ f"per-call {per_call_secret}\n".encode(),
+ f"scoped {scoped_secret}\n".encode(),
+ ],
+ return_code=7,
+ )
+
+ async def remove_handoff(_remote_path: str) -> None:
+ return None
+
+ async def create_subprocess(
+ *args: object,
+ **_kwargs: object,
+ ) -> _BufferedComposeProcess | _BufferedAndStreamedComposeProcess:
+ subprocess_commands.append(args)
+ if 'umask 077; cat > "$1"' in args:
+ return handoff_process
+ if "chmod" in args or "chown" in args:
+ return _BufferedComposeProcess(stdout=b"", return_code=0)
+ return main_process
+
+ async def outer_callback(text: str, stream: str) -> None:
+ callback_events.append(("outer", text, stream))
+
+ async def inner_callback(text: str, stream: str) -> None:
+ callback_events.append(("inner", text, stream))
+
+ monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with (
+ environment.scoped_exec_env({"SCOPED_TOKEN": scoped_secret}),
+ environment.scoped_output_callback(outer_callback),
+ environment.scoped_output_callback(inner_callback),
+ ):
+ return await environment.exec(
+ "emit-handoff-output",
+ env={"PER_CALL_TOKEN": per_call_secret},
+ )
+
+ result = asyncio.run(exercise())
+ marker = _collision_safe_redaction_marker(secrets)
+ expected = f"persistent {marker}\nper-call {marker}\nscoped {marker}\n"
+ outer_chunks = [(text, stream) for label, text, stream in callback_events if label == "outer"]
+ inner_chunks = [(text, stream) for label, text, stream in callback_events if label == "inner"]
+ callback_output = "".join(text for text, _stream in outer_chunks)
+
+ assert callback_output == "".join(text for text, _stream in inner_chunks) == result.stdout == expected
+ assert outer_chunks == inner_chunks
+ assert [label for label, _text, _stream in callback_events] == [
+ label for _chunk in outer_chunks for label in ("outer", "inner")
+ ]
+ assert result.return_code == 7
+ assert {stream for _text, stream in outer_chunks} == {"stdout"}
+ assert handoff_process.communicate_inputs == []
+ handoff_script = bytes(handoff_process.stdin.data).decode("utf-8")
+ assert all(secret in handoff_script for secret in secrets)
+ rendered_commands = "\n".join(" ".join(str(argument) for argument in command) for command in subprocess_commands)
+ for secret in secrets:
+ assert secret not in callback_output
+ assert secret not in (result.stdout or "")
+ assert secret not in rendered_commands
+
+
+@pytest.mark.parametrize(
+ "proxy_uri",
+ [
+ "https://secure%2Duser:secure%2Dpassword@proxy.invalid:8443",
+ "secure-user:secure-password@proxy.invalid:8443",
+ ],
+ ids=["percent-encoded", "schemeless"],
+)
+def test_secure_public_exec_redacts_detached_proxy_components(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ proxy_uri: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ handoff_process = _BufferedComposeProcess(stdout=b"", return_code=0)
+ main_process = _BufferedAndStreamedComposeProcess(
+ [b"proxy rejected secure-user with secure-password\n"],
+ return_code=7,
+ )
+ callback_chunks: list[str] = []
+
+ async def remove_handoff(_remote_path: str) -> None:
+ return None
+
+ async def create_subprocess(
+ *args: object,
+ **_kwargs: object,
+ ) -> _BufferedComposeProcess | _BufferedAndStreamedComposeProcess:
+ if 'umask 077; cat > "$1"' in args or "chmod" in args or "chown" in args:
+ return handoff_process
+ return main_process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ "emit-proxy-failure",
+ env={"HTTPS_PROXY": proxy_uri},
+ )
+
+ result = asyncio.run(exercise())
+ rendered = "".join(callback_chunks) + (result.stdout or "") + (result.stderr or "")
+
+ assert result.return_code == 7
+ assert "secure-user" not in rendered
+ assert "secure-password" not in rendered
+
+
+def test_secure_public_exec_output_limit_redacts_detached_proxy_components(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ diagnostic = b"ordinary-output-" * 10 + b"proxy rejected secure-limit-user with secure-limit-password\n"
+ handoff_process = _BufferedComposeProcess(stdout=b"", return_code=0)
+ main_process = _BufferedAndStreamedComposeProcess([diagnostic, b"overflow"])
+ callback_chunks: list[str] = []
+
+ async def remove_handoff(_remote_path: str) -> None:
+ return None
+
+ async def create_subprocess(
+ *args: object,
+ **_kwargs: object,
+ ) -> _BufferedComposeProcess | _BufferedAndStreamedComposeProcess:
+ if 'umask 077; cat > "$1"' in args or "chmod" in args or "chown" in args:
+ return handoff_process
+ return main_process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ async def contain(
+ contained_process: asyncio.subprocess.Process,
+ communication: asyncio.Task[object],
+ **_kwargs: object,
+ ) -> None:
+ contained_process.terminate()
+ with contextlib.suppress(BaseException):
+ await communication
+
+ monkeypatch.setattr(stream_redaction_module, "MAX_COMMAND_OUTPUT_BYTES", len(diagnostic), raising=False)
+ monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)
+ monkeypatch.setattr(environment, "_contain_main_and_reap_compose", contain)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ "emit-proxy-failure",
+ env={
+ "HTTPS_PROXY": "https://secure-limit-user:secure-limit-password@proxy.invalid:8443",
+ },
+ )
+
+ with pytest.raises(CommandOutputLimitError) as caught:
+ asyncio.run(exercise())
+
+ rendered = "".join(callback_chunks) + str(caught.value)
+ assert "ordinary-output" in "".join(callback_chunks)
+ assert "secure-limit-user" not in rendered
+ assert "secure-limit-password" not in rendered
+
+
+def test_secure_public_exec_timeout_redacts_detached_proxy_components(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ handoff_process = _BufferedComposeProcess(stdout=b"", return_code=0)
+ callback_chunks: list[str] = []
+
+ class OutputThenHangingProcess(_HangingComposeProcess):
+ def __init__(self) -> None:
+ super().__init__(pid=8844)
+ stream = _FeedableStream()
+ stream.feed_data(b"proxy rejected secure-timeout-user with secure-timeout-password\n")
+ self.stdout = stream
+
+ def terminate(self) -> None:
+ self.stdout.feed_eof()
+ super().terminate()
+
+ main_process = OutputThenHangingProcess()
+
+ async def remove_handoff(_remote_path: str) -> None:
+ return None
+
+ async def create_subprocess(
+ *args: object,
+ **_kwargs: object,
+ ) -> _BufferedComposeProcess | OutputThenHangingProcess:
+ if 'umask 077; cat > "$1"' in args or "chmod" in args or "chown" in args:
+ return handoff_process
+ return main_process
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ async def contain(
+ contained_process: OutputThenHangingProcess,
+ communication: asyncio.Task[object],
+ **_kwargs: object,
+ ) -> None:
+ contained_process.terminate()
+ with contextlib.suppress(BaseException):
+ await communication
+
+ monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)
+ monkeypatch.setattr(environment, "_contain_main_and_reap_compose", contain)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> ExecResult:
+ with environment.scoped_output_callback(on_output):
+ return await environment.exec(
+ "emit-proxy-failure",
+ env={
+ "HTTPS_PROXY": "https://secure-timeout-user:secure-timeout-password@proxy.invalid:8443",
+ },
+ timeout_sec=0.05,
+ )
+
+ with pytest.raises(RuntimeError, match="timed out") as caught:
+ asyncio.run(exercise())
+
+ rendered = "".join(callback_chunks) + str(caught.value)
+ assert "proxy rejected" in "".join(callback_chunks)
+ assert "secure-timeout-user" not in rendered
+ assert "secure-timeout-password" not in rendered
+
+
+def test_secure_handoff_scope_scrubs_compose_client_environment_and_resets(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persistent_secret = "persistent-context-scope-secret"
+ per_call_secret = "per-call-context-scope-secret"
+ scoped_secret = "scoped-context-scope-secret"
+ override_secret = "explicit-override-context-secret"
+ handoff_secrets = {persistent_secret, per_call_secret, scoped_secret}
+ all_secrets = handoff_secrets | {override_secret}
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"PERSISTENT_TOKEN": persistent_secret},
+ )
+ environment._compose_task_env = {
+ "COMPOSE_DIRECT": per_call_secret,
+ "COMPOSE_WRAPPED": f"prefix:{persistent_secret}:suffix",
+ }
+ monkeypatch.setenv("PER_CALL_TOKEN", "host-value-hidden-by-active-name")
+ monkeypatch.setenv("SCOPED_TOKEN", "another-host-value-hidden-by-active-name")
+ monkeypatch.setenv("INHERITED_WRAPPED", f"prefix:{scoped_secret}:suffix")
+
+ subprocess_calls: list[tuple[tuple[object, ...], dict[str, object]]] = []
+ callback_chunks: list[str] = []
+ handoff_processes: list[_BufferedComposeProcess] = []
+
+ async def create_subprocess(*args: object, **kwargs: object):
+ subprocess_calls.append((args, kwargs))
+ rendered = " ".join(str(argument) for argument in args)
+ if "emit-context-scope-output" in rendered:
+ raw_output = " ".join(sorted(all_secrets)) + "\n"
+ return _BufferedAndStreamedComposeProcess([raw_output.encode()])
+
+ process = _BufferedComposeProcess(stdout=b"")
+ handoff_processes.append(process)
+ return process
+
+ async def exec_with_explicit_overrides(
+ self,
+ command: str,
+ *,
+ cwd: str | None,
+ timeout_sec: int | None,
+ user: str | int | None,
+ secret_values: set[str] | None = None,
+ ) -> ExecResult:
+ del cwd, timeout_sec, user
+ return await self._run_docker_compose_command(
+ ["exec", "main", command],
+ check=False,
+ on_output=self._output_callback(),
+ env_overrides={
+ "EXPLICIT_OVERRIDE": override_secret,
+ "EXPLICIT_OVERRIDE_WRAPPED": f"prefix:{override_secret}:suffix",
+ },
+ additional_secret_values=secret_values,
+ stop_main_on_interrupt=True,
+ )
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+
+ monkeypatch.setattr(
+ environment,
+ "_exec_without_environment",
+ MethodType(exec_with_explicit_overrides, environment),
+ )
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> tuple[ExecResult, dict[str, str]]:
+ with (
+ environment.scoped_exec_env({"SCOPED_TOKEN": scoped_secret}),
+ environment.scoped_output_callback(on_output),
+ ):
+ result = await environment.exec(
+ "emit-context-scope-output",
+ env={"PER_CALL_TOKEN": per_call_secret},
+ )
+
+ # Outside the secure handoff scope, retain the Harbor compatibility
+ # behavior: Compose gets its ordinary environment and explicit
+ # overrides. This also proves that the successful exec reset its scope.
+ await environment._run_docker_compose_command(
+ ["version"],
+ check=False,
+ env_overrides={"EXPLICIT_OVERRIDE": override_secret},
+ )
+ return result, subprocess_calls[-1][1]["env"] # type: ignore[return-value]
+
+ result, post_scope_environment = asyncio.run(exercise())
+
+ handoff_inputs = [bytes(process.stdin.data) for process in handoff_processes if process.stdin.data]
+ assert handoff_inputs and all(secret.encode() in handoff_inputs[0] for secret in handoff_secrets)
+ assert "".join(callback_chunks) == result.stdout
+ for secret in all_secrets:
+ assert secret not in "".join(callback_chunks)
+ assert secret not in (result.stdout or "")
+
+ scoped_calls = subprocess_calls[:-1]
+ assert len(scoped_calls) == 4 # stdin setup, chmod, command, final removal
+ for arguments, kwargs in scoped_calls:
+ process_environment = kwargs["env"]
+ assert isinstance(process_environment, dict)
+ assert not {"PERSISTENT_TOKEN", "PER_CALL_TOKEN", "SCOPED_TOKEN"} & process_environment.keys()
+ assert "EXPLICIT_OVERRIDE" not in process_environment
+ assert "EXPLICIT_OVERRIDE_WRAPPED" not in process_environment
+ assert all(
+ secret not in value
+ for value in process_environment.values()
+ if isinstance(value, str)
+ for secret in all_secrets
+ )
+ rendered_arguments = " ".join(str(argument) for argument in arguments)
+ assert all(secret not in rendered_arguments for secret in all_secrets)
+
+ assert post_scope_environment["PERSISTENT_TOKEN"] == persistent_secret
+ assert post_scope_environment["COMPOSE_DIRECT"] == per_call_secret
+ assert post_scope_environment["INHERITED_WRAPPED"] == f"prefix:{scoped_secret}:suffix"
+ assert post_scope_environment["EXPLICIT_OVERRIDE"] == override_secret
+
+
+def test_secure_handoff_scope_scrubs_real_compose_client_process_environment(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ secret = "real-child-scope-secret-marker-74891"
+ audit_path = tmp_path / "compose-client-commands.txt"
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ docker_path = bin_dir / "docker"
+ docker_path.write_text(
+ f"""#!{sys.executable}
+import os
+import sys
+
+bad_names = {{"REAL_SCOPE_TOKEN"}} & os.environ.keys()
+bad_values = [name for name, value in os.environ.items() if "scope-secret-marker" in value]
+with open({str(audit_path)!r}, "a", encoding="utf-8") as audit:
+ audit.write(" ".join(sys.argv[1:]) + "\\n")
+sys.stdin.buffer.read()
+if bad_names or bad_values:
+ print("secure handoff leaked into compose client: " + ",".join(sorted(bad_names | set(bad_values))))
+ raise SystemExit(91)
+""",
+ encoding="utf-8",
+ )
+ docker_path.chmod(0o700)
+ monkeypatch.setenv("PATH", str(bin_dir))
+ monkeypatch.setenv("REAL_SCOPE_TOKEN", secret)
+ monkeypatch.setenv("REAL_INHERITED_WRAPPER", f"prefix:{secret}:suffix")
+
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"REAL_SCOPE_TOKEN": secret},
+ )
+ environment._compose_task_env = {"REAL_COMPOSE_WRAPPER": f"prefix:{secret}:suffix"}
+ result = asyncio.run(environment.exec("true"))
-def test_generated_tasks_stage_only_names_and_placeholders(
+ assert result.return_code == 0
+ commands = audit_path.read_text(encoding="utf-8").splitlines()
+ assert len(commands) == 4
+ assert any('umask 077; cat > "$1"' in command for command in commands)
+ assert any(" chmod 600 " in f" {command} " for command in commands)
+ assert any("bash -c" in command and "if ! ." in command for command in commands)
+ assert any(" rm -f -- " in f" {command} " for command in commands)
+
+
+def test_concurrent_secure_handoff_scopes_are_isolated_and_reset(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- monkeypatch.setenv("NVIDIA_API_KEY", _SENTINEL)
- task = generate_harbor_tasks(
- _write_skill(tmp_path),
- tmp_path / "tasks",
- runtime_env={"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"},
- verifier_env={
- "NVIDIA_API_KEY": "${NVIDIA_API_KEY}",
- "OPENAI_API_KEY": "${OPENAI_API_KEY}",
- },
- )[0]
+ secret_a = "concurrent-secure-scope-secret-alpha"
+ secret_b = "concurrent-secure-scope-secret-bravo"
+ monkeypatch.setenv("CONCURRENT_TOKEN_A", secret_a)
+ monkeypatch.setenv("CONCURRENT_TOKEN_B", secret_b)
+ environment = _initialized_secure_docker_environment(tmp_path)
+ setup_barrier = asyncio.Event()
+ setup_count = 0
+ calls: list[dict[str, object]] = []
+ remote_secrets: dict[str, str] = {}
+
+ class CapturingStdin(_WritableStdin):
+ def __init__(self, record: dict[str, object]) -> None:
+ super().__init__()
+ self._record = record
+
+ async def drain(self) -> None:
+ nonlocal setup_count
+ stdin_data = bytes(self.data)
+ self._record["stdin"] = stdin_data
+ if stdin_data:
+ rendered = " ".join(str(argument) for argument in self._record["args"])
+ remote_path = next(
+ token for token in rendered.split() if token.startswith("/tmp/.skillevaluator-exec-env-")
+ )
+ decoded = stdin_data.decode()
+ active_secret = secret_a if secret_a in decoded else secret_b
+ remote_secrets[remote_path] = active_secret
+ setup_count += 1
+ if setup_count == 2:
+ setup_barrier.set()
+ await setup_barrier.wait()
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedComposeProcess:
+ record = {"args": args, "env": dict(kwargs["env"])}
+ calls.append(record)
+ process = _BufferedComposeProcess(stdout=b"")
+ process.stdin = CapturingStdin(record)
+ return process
- staged_text = "\n".join(
- path.read_text(encoding="utf-8", errors="replace") for path in task.rglob("*") if path.is_file()
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> dict[str, str]:
+ results = await asyncio.gather(
+ environment.exec("true", env={"CONCURRENT_TOKEN_A": secret_a}),
+ environment.exec("true", env={"CONCURRENT_TOKEN_B": secret_b}),
+ )
+ assert [result.return_code for result in results] == [0, 0]
+ await environment._run_docker_compose_command(["version"], check=False)
+ return calls[-1]["env"] # type: ignore[return-value]
+
+ post_scope_environment = asyncio.run(exercise())
+
+ assert len(remote_secrets) == 2
+ scoped_calls = calls[:-1]
+ assert len(scoped_calls) == 8
+ for record in scoped_calls:
+ rendered = " ".join(str(argument) for argument in record["args"])
+ remote_path = next(path for path in remote_secrets if path in rendered)
+ active_secret = remote_secrets[remote_path]
+ inactive_secret = secret_b if active_secret == secret_a else secret_a
+ process_environment = record["env"]
+ assert isinstance(process_environment, dict)
+ assert all(active_secret not in value for value in process_environment.values() if isinstance(value, str))
+ assert all(inactive_secret not in value for value in process_environment.values() if isinstance(value, str))
+
+ assert post_scope_environment["CONCURRENT_TOKEN_A"] == secret_a
+ assert post_scope_environment["CONCURRENT_TOKEN_B"] == secret_b
+
+
+def test_secure_handoff_scope_does_not_reinsert_required_compose_dependency(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ secret = "required-compose-dependency-secret"
+ monkeypatch.setenv("REQUIRED_HANDOFF_TOKEN", secret)
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"REQUIRED_HANDOFF_TOKEN": secret},
)
- assert _SENTINEL not in staged_text
- assert 'NVIDIA_API_KEY = "${NVIDIA_API_KEY}"' in staged_text
- assert 'OPENAI_API_KEY = "${OPENAI_API_KEY}"' in staged_text
+ captured_environments: list[dict[str, str]] = []
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ process_environment = dict(kwargs["env"])
+ captured_environments.append(process_environment)
+ if "REQUIRED_HANDOFF_TOKEN" not in process_environment:
+ return _BufferedComposeProcess(
+ stdout=b"required variable REQUIRED_HANDOFF_TOKEN is missing",
+ return_code=17,
+ )
+ return _BufferedComposeProcess(stdout=b"", return_code=0)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
-def test_docker_command_uses_secure_environment_import_path() -> None:
- command = build_harbor_run_command(
- dataset_path="/tmp/dataset",
- agent="opencode",
- job_name="secure-docker",
- env_mode="docker",
+ with pytest.raises(RuntimeError, match="required variable REQUIRED_HANDOFF_TOKEN is missing"):
+ asyncio.run(environment.exec("never-runs"))
+
+ assert captured_environments
+ for process_environment in captured_environments:
+ assert "REQUIRED_HANDOFF_TOKEN" not in process_environment
+ assert all(secret not in value for value in process_environment.values())
+
+
+@pytest.mark.parametrize("handoff_mode", ["stdin", "file"])
+def test_secure_handoff_scope_scrubs_resolved_nvidia_key_from_compose_client(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ handoff_mode: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ resolved_secret = f"resolved-{handoff_mode}-nvidia-secret-93641"
+ if handoff_mode == "stdin":
+ requested_value = NVIDIA_BUILD_STDIN_SENTINEL
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "read_nvidia_build_key_from_stdin",
+ lambda: resolved_secret,
+ )
+ else:
+ requested_value = secure_docker_environment._NVIDIA_BUILD_FILE_SENTINEL
+ key_path = tmp_path / "nvidia-api-key"
+ key_path.write_text(resolved_secret, encoding="utf-8")
+ monkeypatch.setenv(secure_docker_environment._NVIDIA_BUILD_KEY_FILE_ENV, str(key_path))
+
+ monkeypatch.setenv("RESOLVED_NVIDIA_WRAPPER", f"prefix:{resolved_secret}:suffix")
+ environment = _initialized_secure_docker_environment(tmp_path)
+ calls: list[tuple[dict[str, str], _BufferedComposeProcess]] = []
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ process = _BufferedComposeProcess(stdout=b"")
+ calls.append((dict(kwargs["env"]), process))
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ result = asyncio.run(environment.exec("true", env={"NVIDIA_API_KEY": requested_value}))
+
+ assert result.return_code == 0
+ assert len(calls) == 4
+ assert any(resolved_secret.encode() in bytes(process.stdin.data) for _env, process in calls)
+ for process_environment, _process in calls:
+ assert "NVIDIA_API_KEY" not in process_environment
+ assert all(resolved_secret not in value for value in process_environment.values())
+
+
+def test_secure_handoff_setup_error_redacts_merged_secrets(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ persistent_secret = "secure-setup-persistent-secret"
+ per_call_secret = "secure-setup-per-call-secret"
+ secrets = {persistent_secret, per_call_secret}
+ environment = _initialized_secure_docker_environment(
+ tmp_path,
+ persistent_env={"PERSISTENT_TOKEN": persistent_secret},
+ )
+ subprocess_commands: list[tuple[object, ...]] = []
+ subprocess_environments: list[dict[str, str]] = []
+ monkeypatch.setenv("SETUP_ERROR_WRAPPER", f"prefix:{per_call_secret}:suffix")
+
+ async def remove_handoff(_remote_path: str) -> None:
+ return None
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedComposeProcess:
+ subprocess_commands.append(args)
+ subprocess_environments.append(dict(kwargs["env"]))
+ return _BufferedComposeProcess(
+ stdout=f"setup failed {persistent_secret} {per_call_secret}\n".encode(),
+ return_code=7,
+ )
+
+ monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> RuntimeError:
+ try:
+ await environment.exec(
+ "never-runs",
+ env={"PER_CALL_TOKEN": per_call_secret},
+ )
+ except RuntimeError as caught:
+ await environment._run_docker_compose_command(["version"], check=False)
+ return caught
+ raise AssertionError("secure setup failure did not propagate")
+
+ caught = asyncio.run(exercise())
+ detail = str(caught)
+ marker = _collision_safe_redaction_marker(secrets)
+ assert f"setup failed {marker} {marker}\n" in detail
+ for secret in secrets:
+ assert secret not in detail
+ assert secret not in "\n".join(
+ " ".join(str(argument) for argument in command) for command in subprocess_commands
+ )
+ for process_environment in subprocess_environments[:-1]:
+ assert all(secret not in value for value in process_environment.values() for secret in secrets)
+ assert subprocess_environments[-1]["PERSISTENT_TOKEN"] == persistent_secret
+ assert subprocess_environments[-1]["SETUP_ERROR_WRAPPER"] == f"prefix:{per_call_secret}:suffix"
+
+
+def test_secure_handoff_marker_exhaustion_fails_before_container_handoff(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "_REDACTION_SENTINEL_CANDIDATES",
+ (),
+ )
+ monkeypatch.setattr(
+ secure_docker_environment,
+ "unicodedata",
+ SimpleNamespace(category=lambda _candidate: "Cc"),
+ )
+ occupied_private_use = "".join(
+ chr(codepoint)
+ for candidate_range in (
+ range(0xE000, 0xF900),
+ range(0xF0000, 0xFFFFE),
+ range(0x100000, 0x10FFFE),
+ )
+ for codepoint in candidate_range
+ )
+ environment = _initialized_secure_docker_environment(tmp_path)
+ handoff_started = False
+ cleanup_attempted = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ nonlocal handoff_started
+ handoff_started = True
+ return _BufferedComposeProcess()
+
+ async def remove_handoff(_remote_path: str) -> None:
+ nonlocal cleanup_attempted
+ cleanup_attempted = True
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)
+
+ with pytest.raises(RuntimeError, match="Could not construct a collision-safe redaction marker") as caught:
+ asyncio.run(
+ environment.exec(
+ "never-runs",
+ env={"SECRET": occupied_private_use},
+ )
+ )
+
+ assert not handoff_started
+ assert not cleanup_attempted
+ assert occupied_private_use not in str(caught.value)
+
+
+@pytest.mark.parametrize("secret", ["x", "hunter2", "secure-public-callback-failure-secret"])
+@pytest.mark.parametrize("error_type", [_CallbackBaseError, asyncio.CancelledError])
+def test_secure_public_exec_preserves_scoped_callback_failure_and_cleans_up(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ error_type: type[BaseException],
+ secret: str,
+) -> None:
+ monkeypatch.setenv("CALLBACK_FAILURE_WRAPPER", f"prefix:{secret}:suffix")
+ environment = _initialized_secure_docker_environment(tmp_path)
+ main_process = _BufferedAndStreamedComposeProcess(
+ [f"output {secret}\n".encode(), b"unread tail\n"],
+ )
+ callback_errors: list[BaseException] = []
+ callback_chunks: list[str] = []
+ containment_calls: list[bool] = []
+ removed_handoffs: list[str] = []
+ subprocess_environments: list[dict[str, str]] = []
+ handoff_process = _BufferedComposeProcess(stdout=b"", return_code=0)
+
+ async def remove_handoff(remote_path: str) -> None:
+ removed_handoffs.append(remote_path)
+
+ async def create_subprocess(
+ *args: object,
+ **kwargs: object,
+ ) -> _BufferedComposeProcess | _BufferedAndStreamedComposeProcess:
+ subprocess_environments.append(dict(kwargs["env"]))
+ if "version" in args:
+ return _BufferedComposeProcess(stdout=b"", return_code=0)
+ if 'umask 077; cat > "$1"' in args:
+ return handoff_process
+ if "chmod" in args or "chown" in args:
+ return _BufferedComposeProcess(stdout=b"", return_code=0)
+ return main_process
+
+ async def contain_and_reap(
+ _process: object,
+ communication: asyncio.Task[object],
+ *,
+ contain_service_on_interrupt: str | None = None,
+ stop_main_on_interrupt: bool,
+ ) -> None:
+ assert contain_service_on_interrupt is None
+ containment_calls.append(stop_main_on_interrupt)
+ await communication
+
+ async def failing_callback(text: str, _stream: str) -> None:
+ callback_chunks.append(text)
+ error = error_type(f"callback rejected {text}")
+ callback_errors.append(error)
+ raise error
+
+ monkeypatch.setattr(environment, "_remove_handoff", remove_handoff)
+ monkeypatch.setattr(environment, "_contain_main_and_reap_compose", contain_and_reap)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> BaseException:
+ try:
+ with environment.scoped_output_callback(failing_callback):
+ await environment.exec(
+ "emit-output",
+ env={"SECRET": secret},
+ )
+ except BaseException as caught:
+ assert isinstance(caught, error_type)
+ await environment._run_docker_compose_command(["version"], check=False)
+ return caught
+ raise AssertionError("callback failure did not propagate")
+
+ caught = asyncio.run(exercise())
+
+ assert callback_errors and caught is callback_errors[0]
+ marker = _collision_safe_redaction_marker({secret}, include_short=True)
+ expected_callback = f"output {marker}" + ("\n" if len(secret) == 1 else "")
+ assert callback_chunks == [expected_callback]
+ assert str(caught) == f"callback rejected {callback_chunks[0]}"
+ assert secret not in str(caught)
+ assert containment_calls == [True]
+ assert len(removed_handoffs) == 1
+ assert main_process.returncode is not None
+ for process_environment in subprocess_environments[:-1]:
+ for value in process_environment.values():
+ if len(secret) >= 8:
+ assert secret not in value
+ else:
+ assert value != secret
+ assert subprocess_environments[-1]["CALLBACK_FAILURE_WRAPPER"] == f"prefix:{secret}:suffix"
+
+
+def test_secure_handoff_scope_resets_after_cleanup_failure(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ secret = "secure-cleanup-failure-scope-secret"
+ monkeypatch.setenv("CLEANUP_FAILURE_WRAPPER", f"prefix:{secret}:suffix")
+ environment = _initialized_secure_docker_environment(tmp_path)
+ subprocess_environments: list[dict[str, str]] = []
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ subprocess_environments.append(dict(kwargs["env"]))
+ return _BufferedComposeProcess(stdout=b"", return_code=0)
+
+ original_remove_handoff = environment._remove_handoff
+
+ async def failed_remove_handoff(remote_path: str) -> None:
+ await original_remove_handoff(remote_path)
+ raise RuntimeError("forced final handoff cleanup failure")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(environment, "_remove_handoff", failed_remove_handoff)
+
+ async def exercise() -> RuntimeError:
+ try:
+ await environment.exec("true", env={"CLEANUP_SECRET": secret})
+ except RuntimeError as caught:
+ await environment._run_docker_compose_command(["version"], check=False)
+ return caught
+ raise AssertionError("final handoff cleanup failure did not propagate")
+
+ caught = asyncio.run(exercise())
+
+ assert "could not confirm removal of Docker environment handoff" in str(caught)
+ assert len(subprocess_environments) == 5
+ assert all(
+ secret not in value
+ for process_environment in subprocess_environments[:-1]
+ for value in process_environment.values()
+ )
+ assert subprocess_environments[-1]["CLEANUP_FAILURE_WRAPPER"] == f"prefix:{secret}:suffix"
+
+
+@pytest.mark.parametrize("interrupt_mode", ["timeout", "cancel"])
+def test_secure_handoff_scope_covers_containment_and_resets_after_interrupt(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ interrupt_mode: str,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ secret = f"secure-{interrupt_mode}-containment-scope-secret"
+ monkeypatch.setenv("INTERRUPT_SCOPE_WRAPPER", f"prefix:{secret}:suffix")
+ environment = _initialized_secure_docker_environment(tmp_path)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
+ subprocess_calls: list[tuple[tuple[object, ...], dict[str, str]]] = []
+ hanging_process = _HangingComposeProcess(pid=7049)
+
+ async def create_subprocess(*args: object, **kwargs: object):
+ subprocess_calls.append((args, dict(kwargs["env"])))
+ if "never-ending-scope-command" in " ".join(str(argument) for argument in args):
+ return hanging_process
+ return _BufferedComposeProcess(stdout=b"", return_code=0)
+
+ def killpg(pid: int, value: signal.Signals) -> None:
+ assert pid == hanging_process.pid
+ hanging_process.finish(-value)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg)
+
+ async def worker() -> tuple[BaseException, dict[str, str]]:
+ try:
+ await environment.exec(
+ "never-ending-scope-command",
+ env={"INTERRUPT_SECRET": secret},
+ timeout_sec=0.01 if interrupt_mode == "timeout" else None,
+ )
+ except BaseException as caught:
+ await environment._run_docker_compose_command(["version"], check=False)
+ return caught, subprocess_calls[-1][1]
+ raise AssertionError("interrupted secure command did not propagate")
+
+ async def exercise() -> tuple[BaseException, dict[str, str]]:
+ task = asyncio.create_task(worker())
+ await asyncio.wait_for(hanging_process.started.wait(), timeout=1)
+ if interrupt_mode == "cancel":
+ task.cancel()
+ await asyncio.sleep(0)
+ task.cancel()
+ return await asyncio.wait_for(task, timeout=1)
+
+ caught, post_scope_environment = asyncio.run(exercise())
+
+ if interrupt_mode == "timeout":
+ assert isinstance(caught, RuntimeError)
+ assert "timed out" in str(caught)
+ else:
+ assert isinstance(caught, asyncio.CancelledError)
+ assert len(subprocess_calls) == 6
+ scoped_calls = subprocess_calls[:-1]
+ rendered_commands = [" ".join(str(argument) for argument in arguments) for arguments, _env in scoped_calls]
+ assert any(
+ "container ls" in command and "label=com.docker.compose.service=main" in command
+ for command in rendered_commands
)
+ assert any(" rm -f -- " in f" {command} " for command in rendered_commands)
+ assert all(
+ secret not in value
+ for _arguments, process_environment in scoped_calls
+ for value in process_environment.values()
+ )
+ assert post_scope_environment["INTERRUPT_SCOPE_WRAPPER"] == f"prefix:{secret}:suffix"
+
- assert "--env" not in command
- assert command[command.index("--environment-import-path") + 1] == SECURE_DOCKER_ENV_IMPORT_PATH
+def test_runner_additional_secret_values_redact_callback_result_and_error(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ secrets = {
+ "handoff-persistent-secret",
+ "persistent-secret",
+ }
+ additional_secret_values = [
+ "handoff-persistent-secret",
+ "persistent-secret",
+ "handoff-persistent-secret",
+ ]
+ raw_output = " ".join(sorted(secrets)) + "\n"
+ environment = _initialized_docker_environment(tmp_path)
+ callback_chunks: list[list[str]] = [[], []]
+ callback_index = 0
+ captured: list[tuple[tuple[object, ...], dict[str, object]]] = []
+
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedAndStreamedComposeProcess:
+ captured.append((args, kwargs))
+ return _BufferedAndStreamedComposeProcess(
+ [raw_output.encode()],
+ return_code=7,
+ )
+
+ async def on_output(text: str, _stream: str) -> None:
+ callback_chunks[callback_index].append(text)
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ async def exercise() -> tuple[ExecResult, RuntimeError]:
+ nonlocal callback_index
+ result = await environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ check=False,
+ on_output=on_output,
+ additional_secret_values=additional_secret_values,
+ )
+ callback_index = 1
+ with pytest.raises(RuntimeError) as caught:
+ await environment._run_docker_compose_command(
+ ["exec", "main", "emit-output"],
+ on_output=on_output,
+ additional_secret_values=additional_secret_values,
+ )
+ return result, caught.value
+
+ result, error = asyncio.run(exercise())
+ marker = _collision_safe_redaction_marker(secrets)
+ expected = f"{marker} {marker}\n"
+
+ assert "".join(callback_chunks[0]) == result.stdout == expected
+ assert "".join(callback_chunks[1]) == expected
+ assert f"Stdout: {expected}." in str(error)
+ for rendered in (result.stdout or "", "".join(callback_chunks[0]), "".join(callback_chunks[1]), str(error)):
+ for secret in secrets:
+ assert secret not in rendered
+ for arguments, kwargs in captured:
+ rendered_arguments = " ".join(str(argument) for argument in arguments)
+ rendered_environment = "\n".join(f"{name}={value}" for name, value in kwargs["env"].items())
+ for secret in secrets:
+ assert secret not in rendered_arguments
+ assert secret not in rendered_environment
def test_exec_uses_name_only_argv_and_subprocess_override(tmp_path: Path) -> None:
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.environment_dir = tmp_path
- environment.task_env_config = SimpleNamespace(workdir=None, env={"NVIDIA_API_KEY": "${NVIDIA_API_KEY}"})
- environment._persistent_env = {"DATABASE_URL": "old-value"}
- environment.default_user = None
- environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])
+ environment = _initialized_docker_environment(
+ tmp_path,
+ persistent_env={"DATABASE_URL": "old-value"},
+ )
captured: dict[str, object] = {}
async def _capture(
@@ -88,13 +6865,19 @@ async def _capture(
command: list[str],
check: bool = True,
timeout_sec: int | None = None,
+ on_output: object | None = None,
*,
env_overrides=None,
+ additional_secret_values=None,
+ exact_secret_values=None,
stop_main_on_interrupt: bool = False,
) -> ExecResult:
del self, check, timeout_sec
captured["command"] = command
captured["env"] = env_overrides
+ captured["on_output"] = on_output
+ captured["additional_secret_values"] = additional_secret_values
+ captured["exact_secret_values"] = exact_secret_values
captured["stop_main_on_interrupt"] = stop_main_on_interrupt
return ExecResult(stdout="ok", stderr=None, return_code=0)
@@ -121,6 +6904,12 @@ async def _capture(
"NVIDIA_API_KEY": _SENTINEL,
"PLAIN_SETTING": "visible",
}
+ assert captured["on_output"] is None
+ assert captured["exact_secret_values"] == {
+ _SENTINEL,
+ "new-value",
+ }
+ assert captured["additional_secret_values"] is None
assert captured["stop_main_on_interrupt"] is True
@@ -137,32 +6926,20 @@ def test_compose_process_receives_value_only_in_env_and_redacts_failure(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.session_id = "secure-test"
- environment.environment_name = "secure-test"
- environment.environment_dir = tmp_path
- environment._resources_compose_path = None
- environment._mounts_compose_path = None
- environment._use_prebuilt = True
- environment._is_windows_container = False
- environment.extra_docker_compose_paths = []
- environment._network_policy = SimpleNamespace(network_mode="public")
+ environment = _initialized_docker_environment(tmp_path)
environment._compose_env_vars = MethodType(
lambda _self, **_kwargs: {"PATH": "/usr/bin"},
environment,
)
captured: dict[str, object] = {}
- class _Process:
- returncode = 7
-
- async def communicate(self):
- return f"failure included {_SENTINEL}".encode(), None
-
async def _create_subprocess(*args, **kwargs):
captured["args"] = args
captured["env"] = kwargs["env"]
- return _Process()
+ return _BufferedComposeProcess(
+ stdout=f"failure included {_SENTINEL}".encode(),
+ return_code=7,
+ )
monkeypatch.setattr(asyncio, "create_subprocess_exec", _create_subprocess)
with pytest.raises(RuntimeError) as caught:
@@ -176,33 +6953,20 @@ async def _create_subprocess(*args, **kwargs):
assert _SENTINEL not in " ".join(captured["args"])
assert captured["env"]["DATABASE_URL"] == _SENTINEL
assert _SENTINEL not in str(caught.value)
- assert "[REDACTED]" in str(caught.value)
+ assert _marker_for(_SENTINEL) in str(caught.value)
def test_compose_check_false_redacts_success_output(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.session_id = "secure-output-test"
- environment.environment_name = "secure-output-test"
- environment.environment_dir = tmp_path
- environment._resources_compose_path = None
- environment._mounts_compose_path = None
- environment._use_prebuilt = True
- environment._is_windows_container = False
- environment.extra_docker_compose_paths = []
- environment._network_policy = SimpleNamespace(network_mode="public")
+ environment = _initialized_docker_environment(tmp_path)
environment._compose_env_vars = MethodType(lambda _self, **_kwargs: {"PATH": "/usr/bin"}, environment)
- class Process:
- returncode = 0
-
- async def communicate(self) -> tuple[bytes, bytes]:
- return f"stdout {_SENTINEL}".encode(), f"stderr {_SENTINEL}".encode()
-
- async def create_subprocess(*_args: object, **_kwargs: object) -> Process:
- return Process()
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ return _BufferedComposeProcess(
+ stdout=f"stdout {_SENTINEL}\nstderr {_SENTINEL}".encode(),
+ )
monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
result = asyncio.run(
@@ -213,8 +6977,9 @@ async def create_subprocess(*_args: object, **_kwargs: object) -> Process:
)
)
- assert result.stdout == "stdout [REDACTED]"
- assert result.stderr == "stderr [REDACTED]"
+ marker = _marker_for(_SENTINEL)
+ assert result.stdout == f"stdout {marker}\nstderr {marker}"
+ assert result.stderr is None
def test_compose_stdin_handoff_redacts_secret_without_argv_or_environment(
@@ -226,26 +6991,27 @@ def test_compose_stdin_handoff_redacts_secret_without_argv_or_environment(
environment.environment_name = "secure-stdin-test"
environment.environment_dir = tmp_path
environment._resources_compose_path = None
+ environment._env_compose_path = None
environment._mounts_compose_path = None
environment._use_prebuilt = True
environment._is_windows_container = False
environment.extra_docker_compose_paths = []
environment._network_policy = SimpleNamespace(network_mode="public")
+ environment._enable_egress_control = False
+ environment._egress_control_services_compose_path = None
environment._compose_env_vars = MethodType(lambda _self, **_kwargs: {"PATH": "/usr/bin"}, environment)
captured: dict[str, object] = {}
- class Process:
- returncode = 9
-
- async def communicate(self, input_bytes: bytes | None = None) -> tuple[bytes, None]:
- captured["input"] = input_bytes
- return f"failure included {_SENTINEL}".encode(), None
+ process = _BufferedComposeProcess(
+ stdout=f"failure included {_SENTINEL}".encode(),
+ return_code=9,
+ )
- async def create_subprocess(*args: object, **kwargs: object) -> Process:
+ async def create_subprocess(*args: object, **kwargs: object) -> _BufferedComposeProcess:
captured["args"] = args
captured["env"] = kwargs["env"]
captured["stdin"] = kwargs["stdin"]
- return Process()
+ return process
monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
@@ -253,12 +7019,12 @@ async def create_subprocess(*args: object, **kwargs: object) -> Process:
asyncio.run(
environment._run_docker_compose_command(
["exec", "-T", "main", "true"],
- stdin_bytes=b"private-stream-payload",
- redact_values={_SENTINEL},
+ stdin_data=b"private-stream-payload",
+ additional_secret_values={_SENTINEL},
)
)
- assert captured["input"] == b"private-stream-payload"
+ assert bytes(process.stdin.data) == b"private-stream-payload"
assert captured["stdin"] is asyncio.subprocess.PIPE
assert _SENTINEL not in " ".join(str(arg) for arg in captured["args"])
assert _SENTINEL not in captured["env"].values()
@@ -272,34 +7038,21 @@ def test_redact_ignores_short_env_values_that_would_corrupt_loopback_origins() -
origin = "http://127.0.0.1:41927\n"
assert _redact(origin, {"1"}) == origin
assert _redact(origin, {"1", "41927"}) == origin
- assert _redact("token=abcdefgh", {"abcdefgh"}) == "token=[REDACTED]"
+ assert _redact("token=abcdefgh", {"abcdefgh"}) == f"token={_marker_for('abcdefgh')}"
def test_compose_redacts_long_secrets_without_rewriting_short_env_flags(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.session_id = "secure-short-secret-test"
- environment.environment_name = "secure-short-secret-test"
- environment.environment_dir = tmp_path
- environment._resources_compose_path = None
- environment._mounts_compose_path = None
- environment._use_prebuilt = True
- environment._is_windows_container = False
- environment.extra_docker_compose_paths = []
- environment._network_policy = SimpleNamespace(network_mode="public")
+ environment = _initialized_docker_environment(tmp_path)
environment._compose_env_vars = MethodType(lambda _self, **_kwargs: {"PATH": "/usr/bin"}, environment)
long_secret = "abcdefgh"
- class Process:
- returncode = 0
-
- async def communicate(self) -> tuple[bytes, bytes]:
- return b"http://127.0.0.1:41927\n", f"stderr {long_secret}".encode()
-
- async def create_subprocess(*_args: object, **_kwargs: object) -> Process:
- return Process()
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _BufferedComposeProcess:
+ return _BufferedComposeProcess(
+ stdout=f"http://127.0.0.1:41927\nstderr {long_secret}".encode(),
+ )
monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
result = asyncio.run(
@@ -313,67 +7066,72 @@ async def create_subprocess(*_args: object, **_kwargs: object) -> Process:
)
)
- assert result.stdout == "http://127.0.0.1:41927\n"
- assert result.stderr == "stderr [REDACTED]"
+ assert result.stdout == f"http://127.0.0.1:41927\nstderr {_marker_for(long_secret)}"
+ assert result.stderr is None
-def test_compose_cancellation_reaps_process_tree_even_when_repeated(
+@pytest.mark.parametrize("secret", ["x", "hunter2", "abcdefgh"])
+def test_compose_exact_sensitive_values_redact_check_errors_at_every_length(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
+ secret: str,
) -> None:
- from skillevaluator.tier3.harbor import secure_docker_environment
+ environment = _initialized_docker_environment(tmp_path)
+
+ async def create_subprocess(
+ *_args: object,
+ **_kwargs: object,
+ ) -> _BufferedComposeProcess:
+ return _BufferedComposeProcess(
+ stdout=f"failure|{secret}|\n".encode(),
+ return_code=9,
+ )
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.session_id = "secure-cancellation-test"
- environment.environment_name = "secure-cancellation-test"
- environment.environment_dir = tmp_path
- environment._resources_compose_path = None
- environment._mounts_compose_path = None
- environment._use_prebuilt = True
- environment._is_windows_container = False
- environment.extra_docker_compose_paths = []
- environment._network_policy = SimpleNamespace(network_mode="public")
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError) as caught:
+ asyncio.run(
+ environment._run_docker_compose_command(
+ ["version"],
+ check=True,
+ exact_secret_values={secret},
+ )
+ )
+
+ marker = _collision_safe_redaction_marker({secret}, include_short=True)
+ assert secret not in str(caught.value)
+ assert marker in str(caught.value)
+
+
+def test_compose_cancellation_reaps_process_tree_even_when_repeated(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_docker_environment(tmp_path)
environment._compose_env_vars = MethodType(lambda _self, **_kwargs: {"PATH": "/usr/bin"}, environment)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01, raising=False)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01, raising=False)
async def run_cancelled() -> list[str]:
actions: list[str] = []
- communicating = asyncio.Event()
- completed: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
-
- class FakeProcess:
- pid = 4343
- returncode: int | None = None
-
- async def communicate(self) -> tuple[bytes, bytes]:
- communicating.set()
- return await asyncio.shield(completed)
-
- def terminate(self) -> None:
- actions.append("terminate")
-
- def kill(self) -> None:
- actions.append("kill")
- self.returncode = -9
- if not completed.done():
- completed.set_result((b"", b""))
-
- process = FakeProcess()
+ process = _HangingComposeProcess(pid=4343)
- async def create_subprocess(*_args: object, **_kwargs: object) -> FakeProcess:
+ async def create_subprocess(*_args: object, **_kwargs: object) -> _HangingComposeProcess:
return process
def killpg(_pid: int, value: signal.Signals) -> None:
if value == signal.SIGTERM:
- process.terminate()
+ actions.append("terminate")
else:
- process.kill()
+ actions.append("kill")
+ process.finish(-signal.SIGKILL)
monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg, raising=False)
task = asyncio.create_task(environment._run_docker_compose_command(["exec", "main", "sleep", "30"]))
- await asyncio.wait_for(communicating.wait(), timeout=1)
+ await asyncio.wait_for(process.started.wait(), timeout=1)
task.cancel()
await asyncio.sleep(0)
task.cancel()
@@ -392,20 +7150,7 @@ def test_interrupted_exec_stops_main_container_before_reaping_host_client(
) -> None:
from skillevaluator.tier3.harbor import secure_docker_environment
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.session_id = "secure-remote-cancellation-test"
- environment.environment_name = "secure-remote-cancellation-test"
- environment.environment_dir = tmp_path
- environment.default_user = None
- environment.task_env_config = SimpleNamespace(workdir=None, env={})
- environment._persistent_env = {}
- environment._resources_compose_path = None
- environment._mounts_compose_path = None
- environment._use_prebuilt = True
- environment._is_windows_container = False
- environment.extra_docker_compose_paths = []
- environment._network_policy = SimpleNamespace(network_mode="public")
- environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])
+ environment = _initialized_docker_environment(tmp_path)
environment._compose_env_vars = MethodType(lambda _self, **_kwargs: {"PATH": "/usr/bin"}, environment)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
@@ -413,46 +7158,23 @@ def test_interrupted_exec_stops_main_container_before_reaping_host_client(
async def run_cancelled() -> tuple[list[str], list[tuple[object, ...]]]:
actions: list[str] = []
commands: list[tuple[object, ...]] = []
- communicating = asyncio.Event()
- completed: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
-
- class OriginalProcess:
- pid = 4545
- returncode: int | None = None
-
- async def communicate(self) -> tuple[bytes, bytes]:
- communicating.set()
- return await asyncio.shield(completed)
-
- class CleanupProcess:
- pid = 4546
- returncode = 0
-
- def __init__(self, action: str) -> None:
- self.action = action
-
- async def communicate(self) -> tuple[bytes, bytes]:
- actions.append(self.action)
- return b"", b""
+ original = _HangingComposeProcess(pid=4545)
- original = OriginalProcess()
-
- async def create_subprocess(*args: object, **_kwargs: object) -> OriginalProcess | CleanupProcess:
+ async def create_subprocess(*args: object, **_kwargs: object) -> _HangingComposeProcess:
commands.append(args)
- if len(commands) == 1:
- return original
- rendered = " ".join(str(arg) for arg in args)
- return CleanupProcess("container-stop" if " stop " in f" {rendered} " else "container-remove")
+ return original
+
+ async def contain_main() -> None:
+ actions.extend(("container-stop", "container-remove"))
def killpg(pid: int, value: signal.Signals) -> None:
assert pid == original.pid
actions.append(f"host-{value.name.lower()}")
if value == signal.SIGKILL:
- original.returncode = -9
- if not completed.done():
- completed.set_result((b"", b""))
+ original.finish(-signal.SIGKILL)
monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(environment, "_contain_main_container", contain_main)
monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg)
task = asyncio.create_task(
environment.exec(
@@ -461,7 +7183,7 @@ def killpg(pid: int, value: signal.Signals) -> None:
timeout_sec=0.01 if interrupt_mode == "timeout" else None,
)
)
- await asyncio.wait_for(communicating.wait(), timeout=1)
+ await asyncio.wait_for(original.started.wait(), timeout=1)
if interrupt_mode == "cancel":
task.cancel()
await asyncio.sleep(0)
@@ -475,13 +7197,10 @@ def killpg(pid: int, value: signal.Signals) -> None:
actions, commands = asyncio.run(run_cancelled())
- assert len(commands) == 3
+ assert len(commands) == 1
rendered_original = " ".join(str(arg) for arg in commands[0])
- rendered_cleanup = " ".join(str(arg) for arg in commands[1])
assert ".skillevaluator-exec-" not in rendered_original
assert "SKILLEVALUATOR_EXEC_TOKEN" not in rendered_original
- assert rendered_cleanup.endswith("stop --timeout 0 main")
- assert " ".join(str(arg) for arg in commands[2]).endswith("rm --force --stop --volumes main")
assert actions.index("container-stop") < actions.index("host-sigterm")
assert actions.index("container-remove") < actions.index("host-sigterm")
@@ -492,20 +7211,7 @@ def test_exec_cancelled_during_process_creation_still_stops_main_container(
) -> None:
from skillevaluator.tier3.harbor import secure_docker_environment
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.session_id = "secure-creation-cancellation-test"
- environment.environment_name = "secure-creation-cancellation-test"
- environment.environment_dir = tmp_path
- environment.default_user = None
- environment.task_env_config = SimpleNamespace(workdir=None, env={})
- environment._persistent_env = {}
- environment._resources_compose_path = None
- environment._mounts_compose_path = None
- environment._use_prebuilt = True
- environment._is_windows_container = False
- environment.extra_docker_compose_paths = []
- environment._network_policy = SimpleNamespace(network_mode="public")
- environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])
+ environment = _initialized_docker_environment(tmp_path)
environment._compose_env_vars = MethodType(lambda _self, **_kwargs: {"PATH": "/usr/bin"}, environment)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
@@ -517,50 +7223,28 @@ async def run_cancelled() -> tuple[list[str], list[tuple[object, ...]]]:
release_creation = asyncio.Event()
stop_started = asyncio.Event()
release_stop = asyncio.Event()
- completed: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
-
- class OriginalProcess:
- pid = 4645
- returncode: int | None = None
-
- async def communicate(self) -> tuple[bytes, bytes]:
- return await asyncio.shield(completed)
-
- class CleanupProcess:
- pid = 4646
- returncode = 0
-
- def __init__(self, action: str) -> None:
- self.action = action
+ original = _HangingComposeProcess(pid=4645)
- async def communicate(self) -> tuple[bytes, bytes]:
- actions.append(f"{self.action}-started")
- if self.action == "container-stop":
- stop_started.set()
- await release_stop.wait()
- actions.append(f"{self.action}-finished")
- return b"", b""
-
- original = OriginalProcess()
-
- async def create_subprocess(*args: object, **_kwargs: object) -> OriginalProcess | CleanupProcess:
+ async def create_subprocess(*args: object, **_kwargs: object) -> _HangingComposeProcess:
commands.append(args)
- if len(commands) == 1:
- creation_started.set()
- await release_creation.wait()
- return original
- rendered = " ".join(str(arg) for arg in args)
- return CleanupProcess("container-stop" if " stop " in f" {rendered} " else "container-remove")
+ creation_started.set()
+ await release_creation.wait()
+ return original
+
+ async def contain_main() -> None:
+ actions.append("container-stop-started")
+ stop_started.set()
+ await release_stop.wait()
+ actions.extend(("container-stop-finished", "container-remove-finished"))
def killpg(pid: int, value: signal.Signals) -> None:
assert pid == original.pid
actions.append(f"host-{value.name.lower()}")
if value == signal.SIGKILL:
- original.returncode = -9
- if not completed.done():
- completed.set_result((b"", b""))
+ original.finish(-signal.SIGKILL)
monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(environment, "_contain_main_container", contain_main)
monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg)
task = asyncio.create_task(environment.exec("sleep 30", env={"NVIDIA_API_KEY": "creation-secret"}))
await asyncio.wait_for(creation_started.wait(), timeout=1)
@@ -577,9 +7261,7 @@ def killpg(pid: int, value: signal.Signals) -> None:
actions, commands = asyncio.run(run_cancelled())
- assert len(commands) == 3
- assert " ".join(str(arg) for arg in commands[1]).endswith("stop --timeout 0 main")
- assert " ".join(str(arg) for arg in commands[2]).endswith("rm --force --stop --volumes main")
+ assert len(commands) == 1
assert actions.index("container-remove-finished") < actions.index("host-sigterm")
@@ -591,16 +7273,7 @@ def test_interrupted_exec_fails_closed_when_main_container_containment_fails(
) -> None:
from skillevaluator.tier3.harbor import secure_docker_environment
- environment = object.__new__(SkillEvaluatorDockerEnvironment)
- environment.session_id = "secure-stop-failure-test"
- environment.environment_name = "secure-stop-failure-test"
- environment.environment_dir = tmp_path
- environment._resources_compose_path = None
- environment._mounts_compose_path = None
- environment._use_prebuilt = True
- environment._is_windows_container = False
- environment.extra_docker_compose_paths = []
- environment._network_policy = SimpleNamespace(network_mode="public")
+ environment = _initialized_docker_environment(tmp_path)
environment._compose_env_vars = MethodType(lambda _self, **_kwargs: {"PATH": "/usr/bin"}, environment)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
@@ -608,39 +7281,23 @@ def test_interrupted_exec_fails_closed_when_main_container_containment_fails(
async def run_timeout() -> tuple[list[tuple[object, ...]], list[signal.Signals]]:
commands: list[tuple[object, ...]] = []
signals: list[signal.Signals] = []
- communicating = asyncio.Event()
- completed: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
-
- class OriginalProcess:
- pid = 4745
- returncode: int | None = None
+ original = _HangingComposeProcess(pid=4745)
- async def communicate(self) -> tuple[bytes, bytes]:
- communicating.set()
- return await asyncio.shield(completed)
-
- class FailedStopProcess:
- pid = 4746
- returncode = 1
-
- async def communicate(self) -> tuple[bytes, bytes]:
- return b"stop failed", b""
-
- original = OriginalProcess()
-
- async def create_subprocess(*args: object, **_kwargs: object) -> OriginalProcess | FailedStopProcess:
+ async def create_subprocess(*args: object, **_kwargs: object) -> _HangingComposeProcess:
commands.append(args)
- return original if len(commands) == 1 else FailedStopProcess()
+ return original
+
+ async def contain_main() -> None:
+ raise PermissionError("raw Docker containment denied")
def killpg(pid: int, value: signal.Signals) -> None:
assert pid == original.pid
signals.append(value)
if value == signal.SIGKILL:
- original.returncode = -9
- if not completed.done():
- completed.set_result((b"", b""))
+ original.finish(-signal.SIGKILL)
monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(environment, "_contain_main_container", contain_main)
monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg)
task = asyncio.create_task(
environment._run_docker_compose_command(
@@ -649,19 +7306,24 @@ def killpg(pid: int, value: signal.Signals) -> None:
stop_main_on_interrupt=True,
)
)
- await asyncio.wait_for(communicating.wait(), timeout=1)
+ await asyncio.wait_for(original.started.wait(), timeout=1)
if interrupt_mode == "cancel":
task.cancel()
await asyncio.sleep(0)
task.cancel()
- with pytest.raises(RuntimeError, match="main task container containment could not be confirmed"):
- await task
+ with pytest.raises(asyncio.CancelledError) as caught:
+ await task
+ else:
+ with pytest.raises(RuntimeError, match="timed out") as caught:
+ await task
+ assert caught.value.__cause__ is not None
+ assert "containment" in str(caught.value.__cause__)
+ assert any("containment could not be confirmed" in note for note in caught.value.__notes__)
return commands, signals
commands, signals = asyncio.run(run_timeout())
- assert len(commands) >= 2
- assert " ".join(str(arg) for arg in commands[1]).endswith("stop --timeout 0 main")
+ assert len(commands) == 1
assert signals == [signal.SIGTERM, signal.SIGKILL]
@@ -705,12 +7367,743 @@ class FakeProcess:
finished_within_bound = cleanup in done
release.set()
await communication
- await cleanup
+ with pytest.raises(RuntimeError, match="could not confirm Docker client process termination"):
+ await cleanup
return finished_within_bound
assert asyncio.run(run_cleanup()) is True
+def test_raw_docker_creation_cancellation_resolves_process_race_and_reaps_client(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
+
+ async def exercise() -> list[signal.Signals]:
+ creation_started = asyncio.Event()
+ release_creation = asyncio.Event()
+ communication_done: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
+ signals: list[signal.Signals] = []
+
+ class RawProcess:
+ pid = 8451
+ returncode: int | None = None
+
+ async def communicate(self) -> tuple[bytes, bytes]:
+ return await asyncio.shield(communication_done)
+
+ process = RawProcess()
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> RawProcess:
+ creation_started.set()
+ try:
+ await release_creation.wait()
+ except asyncio.CancelledError:
+ await release_creation.wait()
+ return process
+
+ def killpg(pid: int, value: signal.Signals) -> None:
+ assert pid == process.pid
+ signals.append(value)
+ if value == signal.SIGKILL:
+ process.returncode = -9
+ if not communication_done.done():
+ communication_done.set_result((b"", b""))
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg)
+ task = asyncio.create_task(environment._run_trusted_docker_command(["version"]))
+ await asyncio.wait_for(creation_started.wait(), timeout=1)
+ task.cancel()
+ await asyncio.sleep(0)
+ task.cancel()
+ release_creation.set()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, timeout=1)
+ assert process.returncode == -9
+ return signals
+
+ assert asyncio.run(exercise()) == [signal.SIGTERM, signal.SIGKILL]
+
+
+def test_raw_docker_creation_timeout_is_total_deadline_bounded_with_late_reaper(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ monkeypatch.setattr(secure_docker_environment, "_RAW_DOCKER_COMMAND_TIMEOUT_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_RAW_LIFECYCLE_TOTAL_TIMEOUT_SECONDS", 0.03)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
+
+ async def exercise() -> tuple[float, list[signal.Signals]]:
+ creation_started = asyncio.Event()
+ creation_cancelled = asyncio.Event()
+ release_creation = asyncio.Event()
+ process_reaped = asyncio.Event()
+ communication_done: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
+ signals: list[signal.Signals] = []
+
+ class RawProcess:
+ pid = 8453
+ returncode: int | None = None
+
+ async def communicate(self) -> tuple[bytes, bytes]:
+ return await asyncio.shield(communication_done)
+
+ process = RawProcess()
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> RawProcess:
+ creation_started.set()
+ try:
+ await release_creation.wait()
+ except asyncio.CancelledError:
+ creation_cancelled.set()
+ await release_creation.wait()
+ return process
+
+ def killpg(pid: int, value: signal.Signals) -> None:
+ assert pid == process.pid
+ signals.append(value)
+ if value == signal.SIGKILL:
+ process.returncode = -9
+ if not communication_done.done():
+ communication_done.set_result((b"", b""))
+ process_reaped.set()
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg)
+ started_at = asyncio.get_running_loop().time()
+ with (
+ secure_docker_environment._raw_lifecycle_deadline_scope(),
+ pytest.raises(RuntimeError, match="trusted Docker client creation timed out") as caught,
+ ):
+ await environment._run_trusted_docker_command(["version"])
+ elapsed = asyncio.get_running_loop().time() - started_at
+ assert creation_started.is_set()
+ assert creation_cancelled.is_set()
+ assert caught.value.__cause__ is not None
+ assert any("late-process reaper" in note for note in caught.value.__cause__.__notes__)
+ release_creation.set()
+ await asyncio.wait_for(process_reaped.wait(), timeout=1)
+ await asyncio.sleep(0)
+ return elapsed, signals
+
+ elapsed, signals = asyncio.run(exercise())
+
+ assert elapsed < 0.1
+ assert signals == [signal.SIGTERM, signal.SIGKILL]
+
+
+def test_raw_docker_command_timeout_kills_and_reaps_client(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skillevaluator.tier3.harbor import secure_docker_environment
+
+ environment = _initialized_secure_docker_environment(tmp_path)
+ monkeypatch.setattr(secure_docker_environment, "_RAW_DOCKER_COMMAND_TIMEOUT_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_TERMINATE_SECONDS", 0.01)
+ monkeypatch.setattr(secure_docker_environment, "_COMPOSE_KILL_SECONDS", 0.01)
+
+ async def exercise() -> list[signal.Signals]:
+ communication_done: asyncio.Future[tuple[bytes, bytes]] = asyncio.get_running_loop().create_future()
+ signals: list[signal.Signals] = []
+
+ class RawProcess:
+ pid = 8452
+ returncode: int | None = None
+
+ async def communicate(self) -> tuple[bytes, bytes]:
+ return await asyncio.shield(communication_done)
+
+ process = RawProcess()
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> RawProcess:
+ return process
+
+ def killpg(pid: int, value: signal.Signals) -> None:
+ assert pid == process.pid
+ signals.append(value)
+ if value == signal.SIGKILL:
+ process.returncode = -9
+ if not communication_done.done():
+ communication_done.set_result((b"", b""))
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+ monkeypatch.setattr(secure_docker_environment.os, "killpg", killpg)
+ with pytest.raises(RuntimeError, match="trusted Docker client command timed out"):
+ await environment._run_trusted_docker_command(["version"])
+ assert process.returncode == -9
+ return signals
+
+ assert asyncio.run(exercise()) == [signal.SIGTERM, signal.SIGKILL]
+
+
+@pytest.mark.integration
+def test_real_docker_main_sidecar_stdin_streaming_and_redaction(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ request: pytest.FixtureRequest,
+) -> None:
+ docker_info = subprocess.run(
+ ["docker", "info", "--format", "{{.OSType}}"],
+ check=False,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ if docker_info.returncode != 0 or docker_info.stdout.strip() != "linux":
+ pytest.skip("requires a running Linux Docker daemon")
+
+ environment_dir = tmp_path / "real-sidecar-environment"
+ environment_dir.mkdir()
+ helper_image = "alpine:3.20"
+ safe_host_config = "safe-host-config-43127"
+ compose_path = environment_dir / "docker-compose.yaml"
+ compose_content = (
+ 'version: "3.8"\n'
+ "services:\n"
+ " helper:\n"
+ " image: ${HELPER_IMAGE:?required}\n"
+ " environment:\n"
+ " SAFE_HOST_CONFIG: ${SAFE_HOST_CONFIG:?required}\n"
+ ' command: ["sh", "-c", "trap : TERM INT; while :; do sleep 3600; done"]\n'
+ " observer:\n"
+ " image: alpine:3.20\n"
+ ' command: ["sh", "-c", "trap : TERM INT; while :; do sleep 3600; done"]\n'
+ )
+ protected_compose_content = compose_content.replace(
+ " SAFE_HOST_CONFIG: ${SAFE_HOST_CONFIG:?required}\n",
+ " SAFE_HOST_CONFIG: ${SAFE_HOST_CONFIG:?required}\n API_TOKEN: ${API_TOKEN:?required}\n",
+ )
+ compose_path.write_text(compose_content, encoding="utf-8")
+ project = f"skillevaluator-sidecar-{uuid.uuid4().hex[:10]}"
+ cleanup_compose = [
+ "docker",
+ "compose",
+ "--project-name",
+ project,
+ "--project-directory",
+ str(environment_dir),
+ "-f",
+ str(compose_path),
+ ]
+
+ def emergency_cleanup() -> None:
+ compose_path.write_text(compose_content, encoding="utf-8")
+ subprocess.run(
+ [*cleanup_compose, "down", "--remove-orphans", "--volumes"],
+ check=False,
+ capture_output=True,
+ env={
+ **os.environ,
+ "HELPER_IMAGE": helper_image,
+ "SAFE_HOST_CONFIG": safe_host_config,
+ },
+ timeout=60,
+ )
+
+ request.addfinalizer(emergency_cleanup)
+
+ main_persistent_secret = "real-main-persistent-secret-21679"
+ main_task_secret = "real-main-task-secret-32780"
+ main_scoped_secret = "real-main-scoped-secret-43891"
+ main_exec_secret = "real-main-exec-secret-54902"
+ sidecar_secret = "real-sidecar-explicit-secret-65013"
+ sidecar_reused_secret = "real-sidecar-reused-secret-76124"
+ sidecar_control_environment = {
+ "PATH": "/sidecar-only-bin",
+ "HOME": "/sidecar-only-home",
+ "DOCKER_HOST": "tcp://sidecar-only.invalid:2376",
+ "DOCKER_CONFIG": "/sidecar-only-docker-config",
+ "COMPOSE_FILE": "/sidecar-only-compose.yaml",
+ "NORMAL_TOKEN": "real 'quoted' $dollar\nsecond-line-secret",
+ "EMPTY_VALUE": "",
+ }
+ monkeypatch.setenv("REAL_SCOPED_ONLY", main_scoped_secret)
+ monkeypatch.setenv("REAL_MAIN_WRAPPED", f"prefix:{main_persistent_secret}:suffix")
+ monkeypatch.setenv("NVIDIA_API_KEY", NVIDIA_BUILD_STDIN_SENTINEL)
+ monkeypatch.setenv(NVIDIA_BUILD_KEY_STDIN_ENV, "1")
+ monkeypatch.setenv("SKILLEVALUATOR_NVIDIA_API_KEY_FILE", "/tmp/real-main-only-key")
+ monkeypatch.setenv("SAFE_HOST_CONFIG", safe_host_config)
+ monkeypatch.delenv("API_TOKEN", raising=False)
+
+ environment = SkillEvaluatorSecureDockerEnvironment(
+ environment_dir=environment_dir,
+ environment_name="real-sidecar-security",
+ session_id=project,
+ trial_paths=TrialPaths(tmp_path / "real-sidecar-trial"),
+ task_env_config=EnvironmentConfig(
+ docker_image="python:3.13-slim",
+ workdir="/tmp",
+ env={
+ "REAL_TASK_ONLY": main_task_secret,
+ "HELPER_IMAGE": helper_image,
+ },
+ ),
+ persistent_env={"REAL_PERSISTENT_ONLY": main_persistent_secret},
+ )
+ real_create_subprocess = asyncio.create_subprocess_exec
+ all_compose_clients: list[tuple[tuple[object, ...], dict[str, str], asyncio.subprocess.Process]] = []
+ sidecar_clients: list[tuple[tuple[object, ...], dict[str, str], asyncio.subprocess.Process]] = []
+ restore_compose_after_raw_sidecar_start = False
+
+ async def capture_compose_clients(*args: object, **kwargs: object) -> asyncio.subprocess.Process:
+ nonlocal restore_compose_after_raw_sidecar_start
+ process = await real_create_subprocess(*args, **kwargs)
+ all_compose_clients.append((args, dict(kwargs["env"]), process))
+ rendered = tuple(str(argument) for argument in args)
+ if "exec" in rendered and "helper" in rendered:
+ sidecar_clients.append((args, dict(kwargs["env"]), process))
+ if restore_compose_after_raw_sidecar_start and len(rendered) > 2 and rendered[1:3] == ("container", "start"):
+ # Keep the model hostile through raw resolution, containment, and
+ # restart spawn, then restore it before a serialized waiter enters.
+ compose_path.write_text(compose_content, encoding="utf-8")
+ restore_compose_after_raw_sidecar_start = False
+ return process
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", capture_compose_clients)
+
+ async def exercise() -> None:
+ nonlocal restore_compose_after_raw_sidecar_start
+ started = False
+ try:
+ await environment.start(force_build=False)
+ started = True
+
+ main_callback: list[tuple[str, str]] = []
+
+ async def on_main_output(text: str, stream: str) -> None:
+ main_callback.append((text, stream))
+
+ with environment.scoped_output_callback(on_main_output):
+ main_result = await environment.service_exec(
+ "printf 'main-out:%s\\n' \"$REAL_MAIN_EXEC\"; printf 'main-error:%s\\n' \"$REAL_MAIN_EXEC\" >&2",
+ service=MAIN_SERVICE_NAME,
+ env={"REAL_MAIN_EXEC": main_exec_secret},
+ )
+ main_marker = _collision_safe_redaction_marker(
+ {
+ main_persistent_secret,
+ main_exec_secret,
+ helper_image,
+ safe_host_config,
+ },
+ )
+ assert "".join(text for text, _stream in main_callback) == main_result.stdout
+ assert set((main_result.stdout or "").splitlines()) == {
+ f"main-out:{main_marker}",
+ f"main-error:{main_marker}",
+ }
+ assert {stream for _text, stream in main_callback} == {"stdout"}
+
+ sidecar_callback: list[tuple[str, str]] = []
+
+ async def on_sidecar_output(text: str, stream: str) -> None:
+ sidecar_callback.append((text, stream))
+
+ sidecar_command = (
+ "printf 'sidecar-out:%s:%s:%s:%s:%s:%s\\n' \"$REAL_SIDECAR\" "
+ '"$REAL_PERSISTENT_ONLY" "${REAL_TASK_ONLY-unset}" "${REAL_SCOPED_ONLY-unset}" '
+ '"${HELPER_IMAGE-unset}" "$SAFE_HOST_CONFIG"; '
+ "printf 'sidecar-error:%s\\n' \"$REAL_SIDECAR\" >&2; exit 7"
+ )
+ control_hash_command = (
+ "path_hash=$(printf '%s' \"$PATH\" | /bin/busybox sha256sum); path_hash=${path_hash%% *}; "
+ "home_hash=$(printf '%s' \"$HOME\" | /bin/busybox sha256sum); home_hash=${home_hash%% *}; "
+ "host_hash=$(printf '%s' \"$DOCKER_HOST\" | /bin/busybox sha256sum); host_hash=${host_hash%% *}; "
+ "config_hash=$(printf '%s' \"$DOCKER_CONFIG\" | /bin/busybox sha256sum); config_hash=${config_hash%% *}; "
+ "compose_hash=$(printf '%s' \"$COMPOSE_FILE\" | /bin/busybox sha256sum); compose_hash=${compose_hash%% *}; "
+ "normal_hash=$(printf '%s' \"$NORMAL_TOKEN\" | /bin/busybox sha256sum); normal_hash=${normal_hash%% *}; "
+ "empty_hash=$(printf '%s' \"$EMPTY_VALUE\" | /bin/busybox sha256sum); empty_hash=${empty_hash%% *}; "
+ "carrier=gone; /bin/busybox env | /bin/busybox grep -q '^SKILLEVALUATOR_SIDECAR_ENV_' "
+ "&& carrier=found; "
+ "printf 'control:%s:%s:%s:%s:%s:%s:%s carrier=%s\\n' "
+ '"$path_hash" "$home_hash" "$host_hash" "$config_hash" "$compose_hash" "$normal_hash" '
+ '"$empty_hash" "$carrier"'
+ )
+ sidecar_command = sidecar_command.removesuffix("; exit 7") + "; " + control_hash_command + "; exit 7"
+ with (
+ environment.scoped_exec_env({"REAL_SCOPED_ONLY": main_scoped_secret}),
+ environment.scoped_output_callback(on_sidecar_output),
+ ):
+ sidecar_result = await environment.service_exec(
+ sidecar_command,
+ service="helper",
+ env={
+ "REAL_SIDECAR": sidecar_secret,
+ "REAL_PERSISTENT_ONLY": sidecar_reused_secret,
+ **sidecar_control_environment,
+ },
+ )
+ sidecar_marker = _collision_safe_redaction_marker(
+ {
+ sidecar_secret,
+ sidecar_reused_secret,
+ helper_image,
+ safe_host_config,
+ *sidecar_control_environment.values(),
+ },
+ )
+ assert "".join(text for text, _stream in sidecar_callback) == sidecar_result.stdout
+ assert set((sidecar_result.stdout or "").splitlines()) == {
+ f"sidecar-out:{sidecar_marker}:{sidecar_marker}:unset:unset:unset:{sidecar_marker}",
+ f"sidecar-error:{sidecar_marker}",
+ "control:"
+ + ":".join(hashlib.sha256(value.encode()).hexdigest() for value in sidecar_control_environment.values())
+ + " carrier=gone",
+ }
+ assert sidecar_result.return_code == 7
+ assert {stream for _text, stream in sidecar_callback} == {"stdout"}
+ sidecar_identity = await environment.service_exec(
+ 'printf \'%s:%s\' "$PWD" "$(id -u)"',
+ service="helper",
+ cwd="/tmp",
+ user=0,
+ )
+ assert sidecar_identity.stdout == "/tmp:0"
+
+ binary_payload = b"\x00real-binary-stdin\xff\nwith spaces\x00"
+ binary_result = await environment._run_docker_compose_command(
+ [
+ "exec",
+ "-T",
+ MAIN_SERVICE_NAME,
+ "python",
+ "-c",
+ "import hashlib,sys; print(hashlib.sha256(sys.stdin.buffer.read()).hexdigest())",
+ ],
+ check=True,
+ stdin_data=binary_payload,
+ )
+ assert binary_result.stdout == hashlib.sha256(binary_payload).hexdigest() + "\n"
+
+ upload_source = tmp_path / "real-upload-payload.bin"
+ upload_payload = b"\x00real-tar-upload\xff\nwith spaces\x00"
+ upload_source.write_bytes(upload_payload)
+ original_run = environment._run_docker_compose_command
+ cp_failed = False
+
+ async def force_tar_fallback(
+ command: list[str],
+ *args: object,
+ **kwargs: object,
+ ) -> ExecResult:
+ nonlocal cp_failed
+ if command and command[0] == "cp" and not cp_failed:
+ cp_failed = True
+ raise RuntimeError("force real tar-stream fallback")
+ return await original_run(command, *args, **kwargs)
+
+ with monkeypatch.context() as upload_patch:
+ upload_patch.setattr(environment, "_run_docker_compose_command", force_tar_fallback)
+ await environment.upload_file(upload_source, "/tmp/real-uploaded-payload.bin")
+ assert cp_failed is True
+ upload_result = await environment.service_exec(
+ 'python -c \'import hashlib; print(hashlib.sha256(open("/tmp/real-uploaded-payload.bin", "rb").read()).hexdigest())\'',
+ service=MAIN_SERVICE_NAME,
+ )
+ assert upload_result.stdout == hashlib.sha256(upload_payload).hexdigest() + "\n"
+
+ main_download_payload = b"\x00main-download\xff"
+ helper_download_payload = b"\x00helper-download\xfe"
+ main_download_script = (
+ "from pathlib import Path; "
+ f'Path("/tmp/main-download.bin").write_bytes({main_download_payload!r}); '
+ 'Path("/tmp/main-tree/nested").mkdir(parents=True, exist_ok=True); '
+ f'Path("/tmp/main-tree/nested/value.bin").write_bytes({main_download_payload!r})'
+ )
+ await environment.service_exec(
+ f"python -c {shlex.quote(main_download_script)}",
+ service=MAIN_SERVICE_NAME,
+ )
+ await environment.service_exec(
+ "mkdir -p /tmp/helper-tree/nested; "
+ "printf '\\000helper-download\\376' > /tmp/helper-download.bin; "
+ "printf '\\000helper-download\\376' > /tmp/helper-tree/nested/value.bin",
+ service="helper",
+ )
+ main_download_file = tmp_path / "main-downloaded.bin"
+ helper_download_file = tmp_path / "helper-downloaded.bin"
+ main_download_dir = tmp_path / "main-downloaded-tree"
+ helper_download_dir = tmp_path / "helper-downloaded-tree"
+ await environment.service_download_file(
+ "/tmp/main-download.bin",
+ main_download_file,
+ )
+ await environment.service_download_file(
+ "/tmp/helper-download.bin",
+ helper_download_file,
+ service="helper",
+ )
+ await environment.service_download_dir(
+ "/tmp/main-tree",
+ main_download_dir,
+ )
+ await environment.service_download_dir(
+ "/tmp/helper-tree",
+ helper_download_dir,
+ service="helper",
+ )
+ assert main_download_file.read_bytes() == main_download_payload
+ assert helper_download_file.read_bytes() == helper_download_payload
+ assert (main_download_dir / "nested" / "value.bin").read_bytes() == main_download_payload
+ assert (helper_download_dir / "nested" / "value.bin").read_bytes() == helper_download_payload
+
+ def container_id(service: str) -> str:
+ return subprocess.run(
+ [
+ "docker",
+ "ps",
+ "-q",
+ "--filter",
+ f"label=com.docker.compose.project={project}",
+ "--filter",
+ f"label=com.docker.compose.service={service}",
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ ).stdout.strip()
+
+ main_container_id = container_id(MAIN_SERVICE_NAME)
+ observer_container_id = container_id("observer")
+ assert main_container_id and observer_container_id
+
+ def container_generation(service: str) -> tuple[str, str]:
+ identity = container_id(service)
+ started_at = subprocess.run(
+ ["docker", "inspect", "--format", "{{.State.StartedAt}}", identity],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ ).stdout.strip()
+ return identity, started_at
+
+ async def probe_interrupted_process(pid_path: str) -> ExecResult:
+ return await environment.service_exec(
+ "process_state=gone; secret_state=gone; "
+ "for proc_cmdline in /proc/[0-9]*/cmdline; do "
+ '[ -r "$proc_cmdline" ] || continue; '
+ "if tr '\\000' ' ' < \"$proc_cmdline\" 2>/dev/null "
+ '| grep -Fq -- "$PROBE_MARKER"; then process_state=alive; break; fi; done; '
+ "for proc_env in /proc/[0-9]*/environ; do "
+ '[ -r "$proc_env" ] || continue; '
+ "if tr '\\000' '\\n' < \"$proc_env\" 2>/dev/null "
+ "| grep -q '^INTERRUPT_SECRET='; then secret_state=found; break; fi; done; "
+ 'printf \'process=%s secret=%s\' "$process_state" "$secret_state"',
+ service="helper",
+ env={"PROBE_MARKER": pid_path},
+ )
+
+ async def exercise_callback_interrupt(
+ hostile_command: str,
+ interrupt_secret: str,
+ pid_path: str,
+ ) -> ExecResult:
+ callback_error = _CallbackBaseError("real sidecar callback failure")
+ callback_output: list[str] = []
+
+ async def fail_callback(text: str, _stream: str) -> None:
+ callback_output.append(text)
+ raise callback_error
+
+ async def run_with_callback() -> ExecResult:
+ with environment.scoped_output_callback(fail_callback):
+ return await environment.service_exec(
+ hostile_command,
+ service="helper",
+ env={"INTERRUPT_SECRET": interrupt_secret},
+ )
+
+ with pytest.raises(
+ _CallbackBaseError,
+ match="real sidecar callback failure",
+ ) as caught:
+ await run_with_callback()
+ assert caught.value is callback_error
+ assert callback_output
+ assert all(interrupt_secret not in chunk for chunk in callback_output)
+ return await probe_interrupted_process(pid_path)
+
+ for interrupt_mode in ("timeout", "cancel", "callback"):
+ interrupt_secret = f"real-sidecar-{interrupt_mode}-secret-{uuid.uuid4().hex}"
+ pid_path = f"/tmp/sidecar-{interrupt_mode}-{uuid.uuid4().hex}.pid"
+ helper_generation_before = container_generation("helper")
+ hostile_command = (
+ "trap '' TERM INT HUP; sleep 300 & child=$!; "
+ f"printf '%s' \"$child\" > {shlex.quote(pid_path)}; "
+ "printf 'callback-output-boundary-that-exceeds-secret-buffer-length-0123456789\\n'; "
+ 'wait "$child"'
+ )
+
+ if interrupt_mode == "timeout":
+ interrupted = asyncio.create_task(
+ environment.service_exec(
+ hostile_command,
+ service="helper",
+ env={"INTERRUPT_SECRET": interrupt_secret},
+ timeout_sec=0.5,
+ )
+ )
+ await asyncio.sleep(0.15)
+ compose_path.write_text(
+ protected_compose_content,
+ encoding="utf-8",
+ )
+ restore_compose_after_raw_sidecar_start = True
+ concurrent_probe = asyncio.create_task(probe_interrupted_process(pid_path))
+ with pytest.raises(RuntimeError, match="timed out"):
+ await interrupted
+ assert restore_compose_after_raw_sidecar_start is False
+ probe_result = await concurrent_probe
+ elif interrupt_mode == "cancel":
+ interrupted = asyncio.create_task(
+ environment.service_exec(
+ hostile_command,
+ service="helper",
+ env={"INTERRUPT_SECRET": interrupt_secret},
+ )
+ )
+ await asyncio.sleep(0.15)
+ concurrent_probe = asyncio.create_task(probe_interrupted_process(pid_path))
+ interrupted.cancel()
+ await asyncio.sleep(0.05)
+ interrupted.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await interrupted
+ probe_result = await concurrent_probe
+ else:
+ probe_result = await exercise_callback_interrupt(
+ hostile_command,
+ interrupt_secret,
+ pid_path,
+ )
+
+ assert probe_result.stdout == "process=gone secret=gone"
+ helper_generation_after = container_generation("helper")
+ assert helper_generation_after[0] == helper_generation_before[0]
+ assert helper_generation_after[1] != helper_generation_before[1]
+ assert container_id(MAIN_SERVICE_NAME) == main_container_id
+ assert container_id("observer") == observer_container_id
+ main_alive = await environment.service_exec(
+ f"printf main-alive-after-{interrupt_mode}",
+ service=MAIN_SERVICE_NAME,
+ )
+ helper_alive = await environment.service_exec(
+ f"printf helper-alive-after-{interrupt_mode}",
+ service="helper",
+ )
+ assert main_alive.stdout == f"main-alive-after-{interrupt_mode}"
+ assert helper_alive.stdout == f"helper-alive-after-{interrupt_mode}"
+
+ compose_path.write_text(
+ protected_compose_content,
+ encoding="utf-8",
+ )
+ try:
+ await environment.stop_service(MAIN_SERVICE_NAME)
+ finally:
+ compose_path.write_text(compose_content, encoding="utf-8")
+ assert container_id(MAIN_SERVICE_NAME) == ""
+ assert container_id("observer") == observer_container_id
+ post_stop_helper_download = tmp_path / "post-stop-helper-download.bin"
+ await environment.service_download_file(
+ "/tmp/helper-download.bin",
+ post_stop_helper_download,
+ service="helper",
+ )
+ assert post_stop_helper_download.read_bytes() == helper_download_payload
+ finally:
+ compose_path.write_text(compose_content, encoding="utf-8")
+ if started:
+ await environment.stop(delete=True)
+
+ asyncio.run(exercise())
+
+ sidecar_env_calls = [
+ (arguments, process_environment)
+ for arguments, process_environment, _process in sidecar_clients
+ if "REAL_SIDECAR" in " ".join(str(argument) for argument in arguments)
+ ]
+ assert len(sidecar_env_calls) == 1
+ sidecar_arguments, sidecar_process_environment = sidecar_env_calls[0]
+ rendered_sidecar_arguments = " ".join(str(argument) for argument in sidecar_arguments)
+ explicit_sidecar_values = {
+ sidecar_secret,
+ sidecar_reused_secret,
+ *sidecar_control_environment.values(),
+ }
+ carrier_environment = {
+ name: value
+ for name, value in sidecar_process_environment.items()
+ if name.startswith("SKILLEVALUATOR_SIDECAR_ENV_")
+ }
+ assert sidecar_secret not in rendered_sidecar_arguments
+ assert sidecar_reused_secret not in rendered_sidecar_arguments
+ assert set(carrier_environment.values()) == explicit_sidecar_values
+ assert len(carrier_environment) == 2 + len(sidecar_control_environment)
+ assert "REAL_SIDECAR" not in sidecar_process_environment
+ assert sidecar_process_environment.get("REAL_PERSISTENT_ONLY") != sidecar_reused_secret
+ for control_name, target_value in sidecar_control_environment.items():
+ assert sidecar_process_environment.get(control_name) != target_value
+ assert (
+ not {
+ "REAL_TASK_ONLY",
+ "REAL_SCOPED_ONLY",
+ "REAL_MAIN_WRAPPED",
+ "NVIDIA_API_KEY",
+ NVIDIA_BUILD_KEY_STDIN_ENV,
+ "SKILLEVALUATOR_NVIDIA_API_KEY_FILE",
+ }
+ & sidecar_process_environment.keys()
+ )
+ for main_secret in {main_persistent_secret, main_task_secret, main_scoped_secret}:
+ assert all(main_secret not in value for value in sidecar_process_environment.values())
+ assert sidecar_clients
+ assert all(process.returncode is not None for _args, _env, process in sidecar_clients)
+ containment_clients = [
+ (arguments, process_environment, process)
+ for arguments, process_environment, process in all_compose_clients
+ if len(arguments) > 2
+ and str(arguments[1]) == "container"
+ and str(arguments[2]) in {"stop", "kill", "rm", "start"}
+ ]
+ assert containment_clients
+ for arguments, process_environment, process in containment_clients:
+ rendered_arguments = " ".join(str(argument) for argument in arguments)
+ assert all(value not in rendered_arguments for value in explicit_sidecar_values if value)
+ assert not any(name.startswith("SKILLEVALUATOR_SIDECAR_ENV_") for name in process_environment)
+ assert all(value not in process_environment.values() for value in explicit_sidecar_values if value)
+ assert process.returncode is not None
+ leaked_containers = subprocess.run(
+ ["docker", "ps", "-a", "-q", "--filter", f"label=com.docker.compose.project={project}"],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ ).stdout.strip()
+ leaked_networks = subprocess.run(
+ ["docker", "network", "ls", "-q", "--filter", f"label=com.docker.compose.project={project}"],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ ).stdout.strip()
+ assert leaked_containers == ""
+ assert leaked_networks == ""
+
+
@pytest.mark.integration
@pytest.mark.parametrize("stop_mode", ["cancel", "timeout"])
def test_real_docker_interrupted_exec_stops_task_container_only(
@@ -833,16 +8226,13 @@ async def upload_file(self, source_path: Path | str, target_path: str) -> None:
check=True,
)
- environment = object.__new__(ComposeOnlyEnvironment)
- environment.session_id = target_project
- environment.environment_name = target_project
- environment.environment_dir = target_dir
- environment.default_user = None
- environment.task_env_config = SimpleNamespace(workdir=None, env={})
- environment._persistent_env = {}
- environment._is_windows_container = False
- environment._platform = SimpleNamespace(exec_shell_args=lambda command: ["bash", "-c", command])
- environment._compose_env_vars = MethodType(lambda _self, **_kwargs: dict(os.environ), environment)
+ environment = ComposeOnlyEnvironment(
+ environment_dir=target_dir,
+ environment_name=target_project,
+ session_id=target_project,
+ trial_paths=TrialPaths(tmp_path / "interrupted-main-trial"),
+ task_env_config=EnvironmentConfig(docker_image="python:3.13-slim"),
+ )
remote_pid_path = f"/tmp/skillevaluator-test-{uuid.uuid4().hex}.pid"
attack_status_path = f"/tmp/skillevaluator-attack-{uuid.uuid4().hex}.txt"
credential = "credential-must-not-outlive-cancelled-agent-command"
diff --git a/tests/test_issue55_collector_truth.py b/tests/test_issue55_collector_truth.py
index 82feaae7..2e0695dd 100644
--- a/tests/test_issue55_collector_truth.py
+++ b/tests/test_issue55_collector_truth.py
@@ -14,7 +14,10 @@
import pytest
-from skillevaluator.evaluation.tier3_report import render_agent_eval_html_report
+from skillevaluator.evaluation.tier3_report import (
+ agent_eval_result_from_directory,
+ render_agent_eval_html_report,
+)
from skillevaluator.tier3.harbor import collector as collector_module
from skillevaluator.tier3.harbor import report, report_data
from skillevaluator.tier3.harbor.collector import (
@@ -29,6 +32,9 @@
DEFAULT_METRICS,
LEGACY_METRIC_SET,
LEGACY_METRICS,
+ MAX_CUSTOM_METRIC_NAME_BYTES,
+ MAX_CUSTOM_METRICS,
+ extract_custom_metrics,
overall_score,
)
@@ -150,6 +156,7 @@ def _collect(
*,
skip_baseline: bool,
case_ids: list[str],
+ pass_threshold: float = 0.50,
) -> dict[str, object]:
return collect_harbor_results(
skill_name="demo",
@@ -157,6 +164,7 @@ def _collect(
output_dir=tmp_path / "results",
jobs_dir=tmp_path / "jobs",
skip_baseline=skip_baseline,
+ pass_threshold=pass_threshold,
expected_cases=len(case_ids),
expected_case_ids=case_ids,
expected_trials=len(case_ids),
@@ -405,6 +413,76 @@ def test_authoritative_multistep_reward_rejects_invalid_default_constituent_with
assert len(failure["reason"]) <= 2048
+def test_authoritative_final_strategy_accepts_incomplete_intermediate_standard_reward(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ _write_authoritative_multistep_result(
+ job_dir,
+ trial_name,
+ aggregate=_default_reward("case-001", 1.0),
+ step_rewards=[
+ {"metric_set": DEFAULT_METRIC_SET, "accuracy": 0.2},
+ _default_reward("case-001", 1.0),
+ ],
+ )
+
+ [extracted] = _extract_rewards(job_dir)
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ assert extracted.get("evaluation_status") != "failed"
+ assert result["execution_status"] == "succeeded"
+ assert result["agents"]["opencode"]["conditions"]["with_skill"]["scored_attempts"] == 1
+
+
+def test_authoritative_mean_strategy_treats_missing_step_metric_keys_as_zero(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ aggregate = _default_reward("case-001", 0.5)
+ aggregate["accuracy"] = 0.6
+ _write_authoritative_multistep_result(
+ job_dir,
+ trial_name,
+ aggregate=aggregate,
+ step_rewards=[
+ {"metric_set": DEFAULT_METRIC_SET, "accuracy": 0.2},
+ _default_reward("case-001", 1.0),
+ ],
+ )
+
+ [extracted] = _extract_rewards(job_dir)
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ assert extracted.get("evaluation_status") != "failed"
+ assert result["execution_status"] == "succeeded"
+ assert result["agents"]["opencode"]["conditions"]["with_skill"]["scored_attempts"] == 1
+
+
+@pytest.mark.parametrize(("strategy", "aggregate_score"), [("final", 1.0), ("mean", 0.5)])
+@pytest.mark.parametrize("empty_rewards", [None, {}], ids=["missing", "empty"])
+def test_authoritative_strategies_accept_successful_empty_intermediate_verifier_rewards(
+ tmp_path: Path,
+ strategy: str,
+ aggregate_score: float,
+ empty_rewards: dict[str, object] | None,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ trial_dir = _write_authoritative_multistep_result(
+ job_dir,
+ trial_name,
+ aggregate=_default_reward("case-001", aggregate_score),
+ step_rewards=[{}, _default_reward("case-001", 1.0)],
+ )
+ payload = json.loads((trial_dir / "result.json").read_text(encoding="utf-8"))
+ payload["step_results"][0]["verifier_result"] = {} if empty_rewards is None else {"rewards": empty_rewards}
+ (trial_dir / "result.json").write_text(json.dumps(payload), encoding="utf-8")
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ assert result["execution_status"] == "succeeded", strategy
+ assert result["agents"]["opencode"]["conditions"]["with_skill"]["scored_attempts"] == 1
+
+
@pytest.mark.parametrize("status_location", ["result", "verifier", "reward"])
def test_authoritative_root_failed_status_is_never_scored(tmp_path: Path, status_location: str) -> None:
job_dir = tmp_path / "jobs" / "demo-opencode-with"
@@ -700,6 +778,36 @@ def test_custom_only_step_fallback_remains_scoreable(tmp_path: Path) -> None:
assert agent["pass_at_k"]["with_skill"]["rate"] == 1.0
+def test_legacy_reward_alias_step_fallback_remains_custom_only_and_scoreable(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ (trial_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "case-001",
+ "step_results": [
+ {
+ "step_name": "custom",
+ "verifier_result": {"rewards": {"reward": 0.8}},
+ }
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["with_skill"] == {}
+ assert agent["pass_at_k"]["with_skill"]["cases"]["case-001"]["best_score"] == 0.8
+
+
def test_custom_only_authoritative_reward_ignores_standard_judge_sidecar_scan_bound(tmp_path: Path) -> None:
job_dir = tmp_path / "jobs" / "demo-opencode-with"
trial_dir = _write_authoritative_multistep_result(
@@ -720,7 +828,7 @@ def test_custom_only_authoritative_reward_ignores_standard_judge_sidecar_scan_bo
assert agent["custom_with_skill"] == {"domain_quality": 0.8}
-def test_nested_custom_only_step_fallback_preserves_custom_metrics(tmp_path: Path) -> None:
+def test_flat_custom_only_step_fallback_preserves_custom_metrics(tmp_path: Path) -> None:
job_dir = tmp_path / "jobs" / "demo-opencode-with"
trial_name = "case-001__attempt"
trial_dir = job_dir / trial_name
@@ -736,7 +844,7 @@ def test_nested_custom_only_step_fallback_preserves_custom_metrics(tmp_path: Pat
"verifier_result": {
"rewards": {
"metric_set": CUSTOM_ONLY_METRIC_SET,
- "metrics": {"domain_quality": {"score": 0.9}},
+ "domain_quality": 0.9,
"overall": 0.75,
}
},
@@ -757,103 +865,596 @@ def test_nested_custom_only_step_fallback_preserves_custom_metrics(tmp_path: Pat
assert agent["pass_at_k"]["with_skill"]["rate"] == 1.0
-def test_step_fallback_aggregates_each_logical_trial_before_cross_trial_average(
+def test_rootless_multistep_custom_metric_average_zero_fills_missing_keys(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ (trial_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "case-001",
+ "step_results": [
+ {
+ "step_name": "first",
+ "verifier_result": {"rewards": {"overall": 1.0, "quality": 1.0}},
+ },
+ {
+ "step_name": "second",
+ "verifier_result": {"rewards": {"overall": 1.0}},
+ },
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["custom_with_skill"] == {"quality": 0.5}
+ assert agent["pass_at_k"]["with_skill"]["rate"] == 1.0
+
+
+def test_declared_custom_only_step_fallback_cannot_spoof_reserved_metrics(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ (trial_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "case-001",
+ "step_results": [
+ {
+ "step_name": "custom",
+ "verifier_result": {
+ "rewards": {
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ "overall": 0.1,
+ "domain_quality": 0.2,
+ }
+ },
+ }
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["with_skill"] == {}
+ assert agent["custom_with_skill"] == {"domain_quality": 0.2}
+ assert agent["pass_at_k"]["with_skill"]["cases"]["case-001"]["best_score"] == 0.1
+
+
+def test_mixed_step_fallback_uses_one_overall_across_collector_and_report(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ step_rewards = [
+ {
+ "entry_id": "case-001",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ },
+ {
+ "entry_id": "case-001",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 0.2,
+ "domain_quality": 0.4,
+ },
+ ]
+ step_results: list[dict[str, object]] = []
+ for index, reward in enumerate(step_rewards, start=1):
+ step_name = f"step-{index}"
+ verifier_dir = trial_dir / "steps" / step_name / "verifier"
+ verifier_dir.mkdir(parents=True)
+ (verifier_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
+ step_results.append({"step_name": step_name, "verifier_result": {"rewards": reward}})
+ (trial_dir / "result.json").write_text(
+ json.dumps({"trial_name": trial_name, "task_name": "case-001", "step_results": step_results}),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ loaded = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ from skillevaluator.evaluation.tier3_report import _normalize_trials
+
+ [trial] = _normalize_trials(loaded["rewards"], list(DEFAULT_METRICS))
+ assert result["execution_status"] == "succeeded"
+ assert agent["with_skill"] == dict.fromkeys(DEFAULT_METRICS, 1.0)
+ assert summary["overall_score"] == 0.6
+ assert agent["pass_at_k"]["with_skill"]["cases"]["case-001"]["best_score"] == 0.6
+ assert trial["overall"] == 0.6
+
+
+@pytest.mark.parametrize("reverse_rows", [False, True])
+def test_paired_mixed_step_fallback_keeps_report_and_html_aligned_with_logical_overall(
tmp_path: Path,
- monkeypatch: pytest.MonkeyPatch,
+ reverse_rows: bool,
) -> None:
- job_dir = tmp_path / "jobs" / "demo-opencode-with"
- trial_scores = {
- "case-a__attempt": ("case-a", [0.4, 0.4, 1.0]),
- "case-b__attempt": ("case-b", [1.0]),
- }
- for trial_name, (case_id, scores) in trial_scores.items():
+ def write_job(condition: str, standard_score: float, custom_score: float) -> None:
+ job_dir = tmp_path / "jobs" / f"demo-opencode-{condition}"
+ trial_name = "case-001__attempt"
trial_dir = job_dir / trial_name
- trial_dir.mkdir(parents=True)
+ rows = [
+ {
+ "entry_id": "case-001",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, standard_score),
+ },
+ {
+ "entry_id": "case-001",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": custom_score,
+ "domain_quality": custom_score,
+ },
+ ]
+ if reverse_rows:
+ rows.reverse()
step_results: list[dict[str, object]] = []
- for index, score in enumerate(scores, start=1):
+ for index, reward in enumerate(rows, start=1):
step_name = f"step-{index}"
- reward = _default_reward(case_id, score)
- reward["details"] = {"accuracy": {"reason": "accuracy missed" if score < 0.8 else "accurate result"}}
verifier_dir = trial_dir / "steps" / step_name / "verifier"
verifier_dir.mkdir(parents=True)
(verifier_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
step_results.append({"step_name": step_name, "verifier_result": {"rewards": reward}})
(trial_dir / "result.json").write_text(
- json.dumps({"trial_name": trial_name, "task_name": case_id, "step_results": step_results}),
+ json.dumps({"trial_name": trial_name, "task_name": "case-001", "step_results": step_results}),
encoding="utf-8",
)
- _write_complete_job_result(job_dir, list(trial_scores))
+ _write_complete_job_result(job_dir, [trial_name])
- result = _collect(tmp_path, skip_baseline=True, case_ids=["case-a", "case-b"])
+ write_job("with", 1.0, 0.2)
+ write_job("without", 0.6, 0.2)
+ result = _collect(
+ tmp_path,
+ skip_baseline=False,
+ case_ids=["case-001"],
+ pass_threshold=0.75,
+ )
agent = result["agents"]["opencode"]
- assert result["execution_status"] == "succeeded"
- assert agent["conditions"]["with_skill"]["scored_attempts"] == 2
- # The public count remains the number of persisted reward rows for backward
- # compatibility; scoring and pass@k use one aggregate per logical trial.
- assert agent["num_trials_with"] == 4
- assert agent["with_skill"] == dict.fromkeys(DEFAULT_METRICS, 0.8)
- summary = json.loads(
- (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
- )
- assert summary["num_trials"] == 4
- assert len(list((tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json"))) == 4
+ assert agent["pass_at_k"]["with_skill"]["rate"] == 0.0
+ assert agent["pass_at_k"]["with_skill"]["cases"]["case-001"]["best_score"] == 0.6
+ assert agent["pass_at_k"]["without_skill"]["rate"] == 0.0
+ assert agent["pass_at_k"]["without_skill"]["cases"]["case-001"]["best_score"] == 0.4
- # Findings use bounded raw rows for evidence, but their score must remain
- # the complete collector summary even when an entire logical trial is past
- # the report-loader limit.
- monkeypatch.setattr(report_data, "_MAX_TRIALS_PER_CONDITION", 3)
- monkeypatch.setattr(report, "_generate_suggestions_structured", lambda *_args, **_kwargs: [])
- monkeypatch.setattr(report, "_passing_skill_suggestions", lambda *_args, **_kwargs: [])
- assert report.display_findings_report(result, "demo", ["opencode"], tmp_path / "results")
- findings_payload = json.loads((tmp_path / "results" / "opencode" / "findings.json").read_text(encoding="utf-8"))
- [accuracy_finding] = [finding for finding in findings_payload["findings"] if finding["metric"] == "accuracy"]
- assert accuracy_finding["score"] == agent["with_skill"]["accuracy"]
- assert accuracy_finding["severity"] == "ok"
+ skill_dir = tmp_path / "demo-skill"
+ skill_dir.mkdir()
+ report_result = agent_eval_result_from_directory(
+ skill_dir,
+ tmp_path / "results",
+ use_llm_judge=False,
+ )
+ assert report_result is not None
+ report_payload = report_result.metadata["agent_eval"]
+ assert report_payload["overall_score"] == 0.6
+ assert report_payload["overall_lift"] == 0.2
+ # The report verdict remains the documented dimension-only quality gate;
+ # pass@k independently applies the configured per-attempt threshold.
+ assert report_payload["verdict"] == "pass"
+ assert report_payload["agents"]["opencode"]["with_skill"] == 0.6
+ assert report_payload["agents"]["opencode"]["baseline"] == 0.4
+ assert report_payload["agents"]["opencode"]["lift"] == 0.2
+
+ html_path = render_agent_eval_html_report(
+ skill_dir,
+ tmp_path / "results",
+ use_llm_judge=False,
+ )
+ payload_match = re.search(
+ r'',
+ html_path.read_text(encoding="utf-8"),
+ re.DOTALL,
+ )
+ assert payload_match is not None
+ html_payload = json.loads(payload_match.group(1))
+ assert html_payload["overall_score"] == 0.6
+ assert html_payload["overall_lift"] == 0.2
+ assert html_payload["verdict"] == "pass"
+ assert html_payload["pass_at_k"]["with_skill"]["rate"] == 0.0
-def test_saved_fallback_steps_keep_logical_trial_weight_in_findings_and_html(
+@pytest.mark.parametrize("reverse_trials", [False, True])
+def test_paired_mixed_contracts_across_logical_trials_keep_report_aligned_with_collector_overall(
tmp_path: Path,
- monkeypatch: pytest.MonkeyPatch,
+ reverse_trials: bool,
) -> None:
- with_trial_scores = {
- "case-a__attempt": ("case-a", [0.4, 0.4, 1.0]),
- "case-b__attempt": ("case-b", [1.0]),
- }
- without_trial_scores = {
- "case-a__attempt": ("case-a", [0.6]),
- "case-b__attempt": ("case-b", [0.8, 0.8, 0.8]),
- }
-
- def write_job(variant: str, trial_scores: dict[str, tuple[str, list[float]]]) -> None:
- job_dir = tmp_path / "jobs" / f"demo-opencode-{variant}"
- for trial_name, (case_id, scores) in trial_scores.items():
+ def write_job(condition: str, standard_score: float, custom_score: float) -> None:
+ job_dir = tmp_path / "jobs" / f"demo-opencode-{condition}"
+ trials = [
+ (
+ "case-001",
+ {
+ "entry_id": "case-001",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, standard_score),
+ "overall": standard_score,
+ },
+ ),
+ (
+ "case-002",
+ {
+ "entry_id": "case-002",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": custom_score,
+ "domain_quality": custom_score,
+ },
+ ),
+ ]
+ if reverse_trials:
+ trials.reverse()
+ trial_names: list[str] = []
+ for entry_id, reward in trials:
+ trial_name = f"{entry_id}__attempt"
+ trial_names.append(trial_name)
trial_dir = job_dir / trial_name
trial_dir.mkdir(parents=True)
- step_results: list[dict[str, object]] = []
- for index, score in enumerate(scores, start=1):
- step_name = f"step-{index}"
- reward = {
- "entry_id": case_id,
- "metric_set": CUSTOM_ONLY_METRIC_SET,
- "metrics": {"domain_quality": {"score": score}},
- "overall": score,
- }
- if case_id == "case-a":
- reward["custom_details"] = {
- "domain_quality": {
- "reason": "needs improvement" if score < 0.8 else "quality target met",
- }
- }
- verifier_dir = trial_dir / "steps" / step_name / "verifier"
- verifier_dir.mkdir(parents=True)
- (verifier_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
- step_results.append({"step_name": step_name, "verifier_result": {"rewards": reward}})
(trial_dir / "result.json").write_text(
- json.dumps({"trial_name": trial_name, "task_name": case_id, "step_results": step_results}),
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": entry_id,
+ "verifier_result": {"rewards": reward},
+ }
+ ),
encoding="utf-8",
)
- _write_complete_job_result(job_dir, list(trial_scores))
+ _write_reward(job_dir, trial_name, reward)
+ _write_complete_job_result(job_dir, trial_names)
+
+ write_job("with", 1.0, 0.0)
+ write_job("without", 0.6, 0.0)
+ result = _collect(
+ tmp_path,
+ skip_baseline=False,
+ case_ids=["case-001", "case-002"],
+ pass_threshold=0.75,
+ )
+
+ agent = result["agents"]["opencode"]
+ assert agent["pass_at_k"]["with_skill"]["cases"]["case-001"]["best_score"] == 1.0
+ assert agent["pass_at_k"]["with_skill"]["cases"]["case-002"]["best_score"] == 0.0
+ with_summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ without_summary = json.loads(
+ (tmp_path / "results" / "opencode" / "without-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ assert with_summary["overall_score"] == 0.5
+ assert without_summary["overall_score"] == 0.3
+ assert with_summary["mixed_metric_contracts"] is True
+ assert without_summary["mixed_metric_contracts"] is True
+
+ skill_dir = tmp_path / "demo-skill"
+ skill_dir.mkdir()
+ report_result = agent_eval_result_from_directory(
+ skill_dir,
+ tmp_path / "results",
+ use_llm_judge=False,
+ )
+ assert report_result is not None
+ report_payload = report_result.metadata["agent_eval"]
+ assert report_payload["overall_score"] == 0.5
+ assert report_payload["overall_lift"] == 0.2
+ # Verdicts remain the documented dimension-only quality gate even though
+ # the headline score spans both standard and custom logical trials.
+ assert report_payload["verdict"] == "pass"
+ assert report_payload["agents"]["opencode"]["with_skill"] == 0.5
+ assert report_payload["agents"]["opencode"]["baseline"] == 0.3
+ assert report_payload["agents"]["opencode"]["lift"] == 0.2
+
+ html_path = render_agent_eval_html_report(
+ skill_dir,
+ tmp_path / "results",
+ use_llm_judge=False,
+ )
+ payload_match = re.search(
+ r'',
+ html_path.read_text(encoding="utf-8"),
+ re.DOTALL,
+ )
+ assert payload_match is not None
+ html_payload = json.loads(payload_match.group(1))
+ assert html_payload["overall_score"] == 0.5
+ assert html_payload["overall_lift"] == 0.2
+ assert html_payload["verdict"] == "pass"
+
+
+@pytest.mark.parametrize("reverse_trials", [False, True])
+def test_default_v1_v2_mix_keeps_collector_and_report_aligned_with_logical_overall(
+ tmp_path: Path,
+ reverse_trials: bool,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trials: list[tuple[str, dict[str, object]]] = [
+ (
+ "case-v2",
+ {
+ "entry_id": "case-v2",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 0.0),
+ },
+ ),
+ (
+ "case-v1",
+ {
+ "entry_id": "case-v1",
+ "metric_set": LEGACY_METRIC_SET,
+ **dict.fromkeys(LEGACY_METRICS, 1.0),
+ },
+ ),
+ ]
+ if reverse_trials:
+ trials.reverse()
+ trial_names = []
+ for entry_id, reward in trials:
+ trial_name = f"{entry_id}__attempt"
+ trial_names.append(trial_name)
+ _write_reward(job_dir, trial_name, reward)
+ _write_complete_job_result(job_dir, trial_names)
+
+ result = _collect(
+ tmp_path,
+ skip_baseline=True,
+ case_ids=["case-v2", "case-v1"],
+ )
+
+ agent = result["agents"]["opencode"]
+ assert agent["pass_at_k"]["with_skill"]["rate"] == 0.5
+ summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ assert summary["mixed_metric_contracts"] is True
+ assert summary["overall_score"] == 0.5
+ assert summary["scores"] == {
+ "security": 0.0,
+ "skill_execution": 0.5,
+ "skill_efficiency": 0.5,
+ "accuracy": 0.5,
+ "goal_accuracy": 0.5,
+ "behavior_check": 0.5,
+ }
+ assert summary["dimensions"]["security"] == {
+ "score": 0.0,
+ "sources": {"security": 1.0},
+ }
+
+ loaded = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ assert loaded["mixed_metric_contracts_with_skill"] is True
+ skill_dir = tmp_path / "demo-skill"
+ skill_dir.mkdir()
+ report_result = agent_eval_result_from_directory(
+ skill_dir,
+ tmp_path / "results",
+ use_llm_judge=False,
+ )
+ assert report_result is not None
+ report_payload = report_result.metadata["agent_eval"]
+ assert report_payload["overall_score"] == 0.5
+ assert report_payload["agents"]["opencode"]["with_skill"] == 0.5
+ assert report_payload["verdict"] == "fail"
+ dimensions = {
+ dimension["id"]: dimension["with_skill"] for dimension in report_payload["agents"]["opencode"]["dimensions"]
+ }
+ assert dimensions == {
+ "security": 0.0,
+ "correctness": 0.5,
+ "discoverability": 0.5,
+ "effectiveness": 0.5,
+ "efficiency": 0.5,
+ }
+ trials_by_case = {trial["entry_id"]: trial for trial in report_payload["agents"]["opencode"]["trials"]}
+ assert "security" not in trials_by_case["case-v1"]["scores"]
+ assert trials_by_case["case-v1"]["overall"] == 1.0
+
+ html_path = render_agent_eval_html_report(
+ skill_dir,
+ tmp_path / "results",
+ use_llm_judge=False,
+ )
+ payload_match = re.search(
+ r'',
+ html_path.read_text(encoding="utf-8"),
+ re.DOTALL,
+ )
+ assert payload_match is not None
+ html_payload = json.loads(payload_match.group(1))
+ assert html_payload["overall_score"] == 0.5
+ assert html_payload["dimensions"] == report_payload["dimensions"]
+
+
+def test_standard_contract_with_custom_metrics_is_not_marked_as_mixed(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ reward = {
+ **_default_reward("case-001", 1.0),
+ "custom_metrics": {"domain_quality": 0.4},
+ }
+ _write_reward(job_dir, trial_name, reward)
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ assert result["execution_status"] == "succeeded"
+ summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ assert summary["custom_scores"] == {"domain_quality": 0.4}
+ assert summary["mixed_metric_contracts"] is False
+ loaded = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ assert loaded["mixed_metric_contracts_with_skill"] is False
+
+
+def test_result_only_mixed_step_fallback_keeps_rows_for_consistent_overall(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ step_rewards = [
+ {
+ "entry_id": "case-001",
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(DEFAULT_METRICS, 1.0),
+ },
+ {
+ "entry_id": "case-001",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 0.2,
+ "domain_quality": 0.4,
+ },
+ ]
+ (trial_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "case-001",
+ "step_results": [
+ {
+ "step_name": f"step-{index}",
+ "verifier_result": {"rewards": reward},
+ }
+ for index, reward in enumerate(step_rewards, start=1)
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ loaded = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ from skillevaluator.evaluation.tier3_report import _normalize_trials
+
+ [trial] = _normalize_trials(loaded["rewards"], list(DEFAULT_METRICS))
+ assert result["execution_status"] == "succeeded"
+ assert summary["num_reward_rows"] == 2
+ assert summary["overall_score"] == 0.6
+ assert agent["pass_at_k"]["with_skill"]["cases"]["case-001"]["best_score"] == 0.6
+ assert trial["overall"] == 0.6
+
+
+def test_step_fallback_aggregates_each_logical_trial_before_cross_trial_average(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_scores = {
+ "case-a__attempt": ("case-a", [0.4, 0.4, 1.0]),
+ "case-b__attempt": ("case-b", [1.0]),
+ }
+ for trial_name, (case_id, scores) in trial_scores.items():
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ step_results: list[dict[str, object]] = []
+ for index, score in enumerate(scores, start=1):
+ step_name = f"step-{index}"
+ reward = _default_reward(case_id, score)
+ reward["details"] = {"accuracy": {"reason": "accuracy missed" if score < 0.8 else "accurate result"}}
+ verifier_dir = trial_dir / "steps" / step_name / "verifier"
+ verifier_dir.mkdir(parents=True)
+ (verifier_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
+ step_results.append({"step_name": step_name, "verifier_result": {"rewards": reward}})
+ (trial_dir / "result.json").write_text(
+ json.dumps({"trial_name": trial_name, "task_name": case_id, "step_results": step_results}),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, list(trial_scores))
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-a", "case-b"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 2
+ # Public denominators and scoring both count Harbor attempts rather than
+ # physical verifier rows. The raw row count remains in the private summary
+ # so bounded report loading can still detect truncation or missing files.
+ assert agent["num_trials_with"] == 2
+ assert agent["with_skill"] == dict.fromkeys(DEFAULT_METRICS, 0.8)
+ summary = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "summary.json").read_text(encoding="utf-8")
+ )
+ assert summary["num_trials"] == 2
+ assert summary["num_reward_rows"] == 4
+ assert len(list((tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json"))) == 4
+
+ # Findings use bounded raw rows for evidence, but their score must remain
+ # the complete collector summary even when an entire logical trial is past
+ # the report-loader limit.
+ monkeypatch.setattr(report_data, "_MAX_TRIALS_PER_CONDITION", 3)
+ monkeypatch.setattr(report, "_generate_suggestions_structured", lambda *_args, **_kwargs: [])
+ monkeypatch.setattr(report, "_passing_skill_suggestions", lambda *_args, **_kwargs: [])
+ assert report.display_findings_report(result, "demo", ["opencode"], tmp_path / "results")
+ findings_payload = json.loads((tmp_path / "results" / "opencode" / "findings.json").read_text(encoding="utf-8"))
+ [accuracy_finding] = [finding for finding in findings_payload["findings"] if finding["metric"] == "accuracy"]
+ assert accuracy_finding["score"] == agent["with_skill"]["accuracy"]
+ assert accuracy_finding["severity"] == "ok"
+
+
+def test_saved_fallback_steps_keep_logical_trial_weight_in_findings_and_html(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ with_trial_scores = {
+ "case-a__attempt": ("case-a", [0.4, 0.4, 1.0]),
+ "case-b__attempt": ("case-b", [1.0]),
+ }
+ without_trial_scores = {
+ "case-a__attempt": ("case-a", [0.6]),
+ "case-b__attempt": ("case-b", [0.8, 0.8, 0.8]),
+ }
+
+ def write_job(variant: str, trial_scores: dict[str, tuple[str, list[float]]]) -> None:
+ job_dir = tmp_path / "jobs" / f"demo-opencode-{variant}"
+ for trial_name, (case_id, scores) in trial_scores.items():
+ trial_dir = job_dir / trial_name
+ trial_dir.mkdir(parents=True)
+ step_results: list[dict[str, object]] = []
+ for index, score in enumerate(scores, start=1):
+ step_name = f"step-{index}"
+ reward = {
+ "entry_id": case_id,
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "domain_quality": score,
+ "overall": score,
+ }
+ if case_id == "case-a":
+ reward["custom_details"] = {
+ "domain_quality": {
+ "reason": "needs improvement" if score < 0.8 else "quality target met",
+ }
+ }
+ verifier_dir = trial_dir / "steps" / step_name / "verifier"
+ verifier_dir.mkdir(parents=True)
+ (verifier_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
+ step_results.append({"step_name": step_name, "verifier_result": {"rewards": reward}})
+ (trial_dir / "result.json").write_text(
+ json.dumps({"trial_name": trial_name, "task_name": case_id, "step_results": step_results}),
+ encoding="utf-8",
+ )
+ _write_complete_job_result(job_dir, list(trial_scores))
write_job("with", with_trial_scores)
write_job("without", without_trial_scores)
@@ -887,6 +1488,7 @@ def write_job(variant: str, trial_scores: dict[str, tuple[str, list[float]]]) ->
assert domain_finding["score"] == agent["custom_with_skill"]["domain_quality"]
assert domain_finding["severity"] == "ok"
+ monkeypatch.setattr(report_data, "_MAX_TRIALS_PER_CONDITION", 512)
skill_dir = tmp_path / "demo-skill"
skill_dir.mkdir()
report_path = render_agent_eval_html_report(
@@ -910,12 +1512,15 @@ def write_job(variant: str, trial_scores: dict[str, tuple[str, list[float]]]) ->
assert report_payload["agents"]["opencode"]["with_skill"] == 0.8
assert report_payload["agents"]["opencode"]["baseline"] == 0.7
assert report_payload["agents"]["opencode"]["lift"] == 0.1
- assert report_payload["agents"]["opencode"]["num_trials"] == 4
- assert report_payload["agents"]["opencode"]["num_trials_baseline"] == 4
+ assert report_payload["agents"]["opencode"]["num_trials"] == 2
+ assert report_payload["agents"]["opencode"]["num_trials_baseline"] == 2
+ assert len(report_payload["agents"]["opencode"]["trials"]) == 2
+ assert len(report_payload["agents"]["opencode"]["trials_baseline"]) == 2
# A legacy summary has no canonical overall. When its raw rows are
# truncated, the report must show unavailable rather than publish the
# partial first logical trial as a numeric headline.
+ monkeypatch.setattr(report_data, "_MAX_TRIALS_PER_CONDITION", 3)
for variant in ("with-skill", "without-skill"):
summary_path = tmp_path / "results" / "opencode" / variant / "summary.json"
legacy_summary = json.loads(summary_path.read_text(encoding="utf-8"))
@@ -972,7 +1577,10 @@ def write_job(variant: str, trial_scores: dict[str, tuple[str, list[float]]]) ->
),
(
_default_reward("case-001", 0.8),
- [_default_reward("case-001", 0.7), {"overall": 0.9, "domain_quality": 1.0}],
+ [
+ _default_reward("case-001", 0.7),
+ {**_default_reward("case-001", 0.8), "domain_quality": 1.0},
+ ],
{},
),
(
@@ -981,7 +1589,7 @@ def write_job(variant: str, trial_scores: dict[str, tuple[str, list[float]]]) ->
{},
),
],
- ids=("custom-only", "mixed-default-and-custom", "complete-default"),
+ ids=("custom-only", "default-with-custom-metric", "complete-default"),
)
def test_authoritative_multistep_reward_preserves_complete_custom_topologies(
tmp_path: Path,
@@ -1006,47 +1614,217 @@ def test_authoritative_multistep_reward_preserves_complete_custom_topologies(
assert agent["pass_at_k"]["with_skill"]["rate"] == 1.0
-def _create_step_entries(trial_dir: Path, names: list[str], *, directories: bool) -> None:
- steps_dir = trial_dir / "steps"
- steps_dir.mkdir(parents=True, exist_ok=True)
- # Intentionally avoid lexical creation order so correctness cannot depend on
- # whichever directory entry happens to be returned first by the filesystem.
- shuffled_names = names[::2] + list(reversed(names[1::2]))
- for name in shuffled_names:
- path = steps_dir / name
- if directories:
- (path / "verifier").mkdir(parents=True)
- else:
- path.write_text("not a step directory", encoding="utf-8")
+def test_authoritative_root_preserves_flat_numeric_custom_metric(tmp_path: Path) -> None:
+ aggregate: dict[str, object] = {"overall": 0.75, "quality": 0.8}
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ _write_authoritative_multistep_result(
+ job_dir,
+ "case-001__attempt",
+ aggregate=aggregate,
+ step_rewards=[dict(aggregate)],
+ )
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 1
+ assert agent["custom_with_skill"] == {"quality": 0.8}
@pytest.mark.parametrize(
- ("total_candidate_count", "expected_status"),
+ "invalid_surface",
[
- (collector_module._MAX_FAILED_JUDGE_SIDECARS, "succeeded"),
- (collector_module._MAX_FAILED_JUDGE_SIDECARS + 1, "failed"),
+ {"custom_metrics": {"quality": 0.8}},
+ {"metrics": {"quality": 0.8}},
+ {"quality": {"score": 0.8}},
],
- ids=("exact-candidate-limit", "beyond-candidate-limit"),
+ ids=("custom-metrics-container", "metrics-container", "dict-score"),
)
-def test_public_collection_fails_closed_when_sidecar_candidate_limit_is_exceeded(
+def test_authoritative_root_rejects_nonnumeric_reward_shapes(
tmp_path: Path,
- total_candidate_count: int,
- expected_status: str,
+ invalid_surface: dict[str, object],
) -> None:
+ aggregate = {"overall": 0.75, **invalid_surface}
job_dir = tmp_path / "jobs" / "demo-opencode-with"
- trial_dir = _write_authoritative_multistep_result(
+ _write_authoritative_multistep_result(
job_dir,
"case-001__attempt",
- aggregate=_default_reward("case-001", 1.0),
- step_rewards=[_default_reward("case-001", 1.0)],
+ aggregate=aggregate,
+ step_rewards=[dict(aggregate)],
)
- # Only sidecars that actually exist consume the sidecar bound. Empty
- # verifier directories remain compatible with large native step graphs.
- step_count = total_candidate_count
- _create_step_entries(
- trial_dir,
- [f"candidate-{index:04d}" for index in range(step_count)],
- directories=True,
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
+ assert agent["custom_with_skill"] == {}
+
+
+@pytest.mark.parametrize("surface", ["custom_metrics", "metrics"])
+def test_authoritative_root_omits_unsafe_names_from_nested_harbor_reward_details(
+ tmp_path: Path,
+ surface: str,
+) -> None:
+ credential = "ghp_" + ("a" * 36)
+ oversized = "x" * (MAX_CUSTOM_METRIC_NAME_BYTES + 1)
+ invalid = "invalid_quality"
+ raw_metrics: dict[str, object] = {
+ credential: 0.6,
+ oversized: 0.7,
+ invalid: 1.1,
+ "quality": 0.8,
+ }
+ aggregate: dict[str, object] = {"overall": 0.75}
+ aggregate[surface] = (
+ raw_metrics if surface == "custom_metrics" else {name: {"score": score} for name, score in raw_metrics.items()}
+ )
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ _write_authoritative_multistep_result(
+ job_dir,
+ "case-001__attempt",
+ aggregate=aggregate,
+ step_rewards=[dict(aggregate)],
+ )
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ assert result["execution_status"] == "failed"
+ reward_path = next((tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json"))
+ persisted_text = reward_path.read_text(encoding="utf-8")
+ json.loads(persisted_text)
+ assert credential not in persisted_text
+ assert oversized not in persisted_text
+ assert invalid not in persisted_text
+
+
+@pytest.mark.parametrize("root_kind", ["custom-only", "default"])
+def test_authoritative_root_rejects_invalid_constituent_custom_metric_contract(
+ tmp_path: Path,
+ root_kind: str,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ if root_kind == "default":
+ aggregate = _default_reward("case-001", 1.0)
+ step_reward = _default_reward("case-001", 1.0)
+ else:
+ aggregate = {"overall": 0.75, "domain_quality": 0.8}
+ step_reward = {"overall": 0.75}
+ step_reward.update({f"metric_{index:03d}": 1.0 for index in range(MAX_CUSTOM_METRICS + 1)})
+ _write_authoritative_multistep_result(
+ job_dir,
+ "case-001__attempt",
+ aggregate=aggregate,
+ step_rewards=[step_reward],
+ )
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
+ reward_path = next((tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json"))
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert persisted["evaluation_status"] == "failed"
+ assert "custom metric" in json.dumps(persisted["evaluation_errors"]).casefold()
+
+
+def test_authoritative_root_rejects_union_of_valid_constituent_custom_metrics(
+ tmp_path: Path,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ left_count = MAX_CUSTOM_METRICS // 2 + 1
+ right_count = MAX_CUSTOM_METRICS - left_count + 1
+ _write_authoritative_multistep_result(
+ job_dir,
+ "case-001__attempt",
+ aggregate={"overall": 0.75},
+ step_rewards=[
+ {
+ "overall": 0.75,
+ **{f"left_{index:03d}": 1.0 for index in range(left_count)},
+ },
+ {
+ "overall": 0.75,
+ **{f"right_{index:03d}": 1.0 for index in range(right_count)},
+ },
+ ],
+ )
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
+ assert agent["custom_with_skill"] == {}
+
+
+def test_authoritative_root_rejects_reserved_constituent_custom_metric_name(
+ tmp_path: Path,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ _write_authoritative_multistep_result(
+ job_dir,
+ "case-001__attempt",
+ aggregate={"overall": 0.75},
+ step_rewards=[
+ {
+ "overall": 0.75,
+ "custom_metrics": {"security": 0.5, "quality": 0.8},
+ }
+ ],
+ )
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
+ assert agent["custom_with_skill"] == {}
+
+
+def _create_step_entries(trial_dir: Path, names: list[str], *, directories: bool) -> None:
+ steps_dir = trial_dir / "steps"
+ steps_dir.mkdir(parents=True, exist_ok=True)
+ # Intentionally avoid lexical creation order so correctness cannot depend on
+ # whichever directory entry happens to be returned first by the filesystem.
+ shuffled_names = names[::2] + list(reversed(names[1::2]))
+ for name in shuffled_names:
+ path = steps_dir / name
+ if directories:
+ (path / "verifier").mkdir(parents=True)
+ else:
+ path.write_text("not a step directory", encoding="utf-8")
+
+
+@pytest.mark.parametrize(
+ ("total_candidate_count", "expected_status"),
+ [
+ (collector_module._MAX_FAILED_JUDGE_SIDECARS, "succeeded"),
+ (collector_module._MAX_FAILED_JUDGE_SIDECARS + 1, "failed"),
+ ],
+ ids=("exact-candidate-limit", "beyond-candidate-limit"),
+)
+def test_public_collection_fails_closed_when_sidecar_candidate_limit_is_exceeded(
+ tmp_path: Path,
+ total_candidate_count: int,
+ expected_status: str,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_dir = _write_authoritative_multistep_result(
+ job_dir,
+ "case-001__attempt",
+ aggregate=_default_reward("case-001", 1.0),
+ step_rewards=[_default_reward("case-001", 1.0)],
+ )
+ # Only sidecars that actually exist consume the sidecar bound. Empty
+ # verifier directories remain compatible with large native step graphs.
+ step_count = total_candidate_count
+ _create_step_entries(
+ trial_dir,
+ [f"candidate-{index:04d}" for index in range(step_count)],
+ directories=True,
)
for index in range(step_count):
sidecar = trial_dir / "steps" / f"candidate-{index:04d}" / "verifier" / "skill_evaluator_reward.json"
@@ -1342,186 +2120,1035 @@ def test_public_collection_rejects_symlinked_trial_root_without_reading_external
result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
- condition = result["agents"]["opencode"]["conditions"]["with_skill"]
- assert result["execution_status"] == "failed"
- assert condition["scored_attempts"] == 0
- assert result["agents"]["opencode"]["with_skill"] == {}
- assert "symlink" in " ".join(result["execution_errors"]).casefold()
- copied = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "trial.log"
- assert not copied.exists()
- assert marker not in json.dumps(result)
+ condition = result["agents"]["opencode"]["conditions"]["with_skill"]
+ assert result["execution_status"] == "failed"
+ assert condition["scored_attempts"] == 0
+ assert result["agents"]["opencode"]["with_skill"] == {}
+ assert "symlink" in " ".join(result["execution_errors"]).casefold()
+ copied = tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "trial.log"
+ assert not copied.exists()
+ assert marker not in json.dumps(result)
+
+
+def test_single_step_result_fallback_honors_failed_top_level_sidecar(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ trial_dir = job_dir / trial_name
+ _numeric, rich = _failed_judge_artifacts("case-001")
+ trial_dir.mkdir(parents=True)
+ (trial_dir / "result.json").write_text(
+ json.dumps(
+ {
+ "trial_name": trial_name,
+ "task_name": "case-001",
+ "verifier_result": {"rewards": _default_reward("case-001", 1.0)},
+ }
+ ),
+ encoding="utf-8",
+ )
+ verifier_dir = trial_dir / "verifier"
+ verifier_dir.mkdir()
+ (verifier_dir / "skill_evaluator_reward.json").write_text(json.dumps(rich), encoding="utf-8")
+ _write_complete_job_result(job_dir, [trial_name])
+
+ [extracted] = _extract_rewards(job_dir)
+ assert extracted["evaluation_status"] == "failed"
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+ assert result["execution_status"] == "failed"
+ assert result["agents"]["opencode"]["with_skill"] == {}
+
+
+def test_failed_sidecar_overrides_conflicting_reward_status_and_errors(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ secret = "sk-authoritativesidecar123"
+ reward = {
+ **_default_reward("case-001", 1.0),
+ "evaluation_status": "succeeded",
+ "evaluation_errors": {"accuracy": "stale reward error"},
+ }
+ _numeric, rich = _failed_judge_artifacts("case-001", reason=f"provider echoed {secret}")
+ rich["evaluation_status"] = "error"
+ _write_reward(job_dir, trial_name, reward, sidecar=rich)
+ _write_complete_job_result(job_dir, [trial_name])
+
+ [extracted] = _extract_rewards(job_dir)
+ assert extracted["evaluation_status"] == "failed"
+ assert extracted["evaluation_errors"]["accuracy"] == "provider echoed sk-"
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+ assert result["execution_status"] == "failed"
+ assert secret not in " ".join(result["execution_errors"])
+
+
+def test_default_plus_custom_cannot_rescue_an_incomplete_default_reward(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ numeric, rich = _failed_judge_artifacts("case-001")
+ _write_reward(
+ job_dir,
+ trial_name,
+ numeric,
+ sidecar=rich,
+ custom={"overall": 1.0, "domain_quality": 1.0, "details": {"domain_quality": {"reason": "perfect"}}},
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["with_skill"] == {}
+ assert agent["custom_with_skill"] == {}
+ assert agent["pass_at_k"]["with_skill"] == {}
+
+
+def test_custom_only_overall_reward_remains_scoreable(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ trial_name = "case-001__attempt"
+ _write_reward(
+ job_dir,
+ trial_name,
+ {"entry_id": "case-001", "overall": 0.75, "domain_quality": 0.9, "token_efficiency": 0.8},
+ )
+ _write_complete_job_result(job_dir, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["with_skill"] == {}
+ assert agent["custom_with_skill"] == {"domain_quality": 0.9, "token_efficiency": 0.8}
+ assert agent["pass_at_k"]["with_skill"]["rate"] == 1.0
+ assert agent["conditions"]["without_skill"]["execution_status"] == "skipped"
+ persisted = json.loads(
+ (tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json").read_text(
+ encoding="utf-8"
+ )
+ )
+ assert persisted["token_efficiency"] == 0.8
+
+
+def test_mixed_failed_condition_suppresses_published_quality_and_paired_artifacts(tmp_path: Path) -> None:
+ jobs_dir = tmp_path / "jobs"
+ with_job = jobs_dir / "demo-opencode-with"
+ without_job = jobs_dir / "demo-opencode-without"
+
+ _write_reward(with_job, "case-001__attempt", _default_reward("case-001", 0.9))
+ numeric, rich = _failed_judge_artifacts("case-002")
+ _write_reward(with_job, "case-002__attempt", numeric, sidecar=rich)
+ _write_complete_job_result(with_job, ["case-001__attempt", "case-002__attempt"])
+
+ for case_id in ("case-001", "case-002"):
+ _write_reward(without_job, f"{case_id}__attempt", _default_reward(case_id, 0.2))
+ _write_complete_job_result(without_job, ["case-001__attempt", "case-002__attempt"])
+
+ result = _collect(tmp_path, skip_baseline=False, case_ids=["case-001", "case-002"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 1
+ assert agent["conditions"]["without_skill"]["execution_status"] == "succeeded"
+ assert agent["with_skill"] == {}
+ assert agent["custom_with_skill"] == {}
+ assert agent["dimensions_with_skill"] == {}
+ assert agent["pass_at_k"]["with_skill"] == {}
+ assert agent["without_skill"] == dict.fromkeys(DEFAULT_METRICS, 0.2)
+ assert agent["dimensions_without_skill"]
+ assert agent["pass_at_k"]["without_skill"]["rate"] == 0.0
+ assert agent["lift"] == {}
+ assert agent["custom_lift"] == {}
+ assert agent["pass_at_k"]["lift"] == {}
+ assert agent["security_attribution"] == {}
+
+ results_dir = tmp_path / "results" / "opencode"
+ persisted = json.loads((results_dir / "with-skill" / "summary.json").read_text(encoding="utf-8"))
+ assert persisted["scores"] == {}
+ assert persisted["custom_scores"] == {}
+ assert persisted["dimensions"] == {}
+ assert persisted["pass_at_k"] == {}
+ assert not (results_dir / "lift.json").exists()
+ assert not (results_dir / "custom_lift.json").exists()
+ assert not (results_dir / "pass_at_k_lift.json").exists()
+ assert not (results_dir / "security_attribution.json").exists()
+ # Both source artifacts remain available as redacted trial diagnostics.
+ assert (results_dir / "with-skill" / "trials" / "case-001__attempt" / "reward.json").exists()
+ failed_reward = results_dir / "with-skill" / "trials" / "case-002__attempt" / "reward.json"
+ assert json.loads(failed_reward.read_text(encoding="utf-8"))["evaluation_status"] == "failed"
+
+ skill_dir = tmp_path / "demo"
+ skill_dir.mkdir()
+ html = render_agent_eval_html_report(skill_dir, tmp_path / "results", use_llm_judge=False).read_text(
+ encoding="utf-8"
+ )
+ assert "Evaluation incomplete" in html
+ assert "Required judge evaluation failed" in html
+ assert re.findall(r'class="t3-dim-score"[^>]*>([^<]+)', html) == ["N/A"] * 5
+ assert "return raw === null || raw === undefined ? null : Number(raw);" in html
+ payload_match = re.search(
+ r'',
+ html,
+ re.DOTALL,
+ )
+ assert payload_match is not None
+ report_payload = json.loads(payload_match.group(1))
+ for dimension in report_payload["agents"]["opencode"]["dimensions"]:
+ assert dimension["with_skill"] is None
+ assert dimension["score"] is None
+ assert dimension["verdict"] is None
+ assert "0.00" not in str(dimension["explanation"])
+ assert all("0.00" not in bullet for bullet in dimension["reasoning_bullets"])
+
+
+def test_missing_with_job_never_aliases_without_skill_job(tmp_path: Path) -> None:
+ without_job = tmp_path / "jobs" / "demo-opencode-without"
+ trial_name = "case-001__attempt"
+ _write_reward(without_job, trial_name, _default_reward("case-001", 0.2))
+ _write_complete_job_result(without_job, [trial_name])
+
+ result = _collect(tmp_path, skip_baseline=False, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
+ assert agent["with_skill"] == {}
+ assert agent["job_failures"]["with_skill"] == ("Harbor job directory was not created: demo-opencode-with")
+ assert agent["conditions"]["without_skill"]["execution_status"] == "succeeded"
+ assert agent["conditions"]["without_skill"]["scored_attempts"] == 1
+ assert agent["without_skill"] == dict.fromkeys(DEFAULT_METRICS, 0.2)
+ assert not (tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name).exists()
+ assert (tmp_path / "results" / "opencode" / "without-skill" / "trials" / trial_name).exists()
+
+
+@pytest.mark.parametrize("physical_trial_root", ["case-a__x", "CASE-A__x"])
+def test_persisted_step_trial_name_never_collides_with_physical_trial_root(
+ tmp_path: Path,
+ physical_trial_root: str,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ _write_reward(job_dir, "case-a/steps/x", _default_reward("case-a", 0.2))
+ _write_reward(job_dir, physical_trial_root, _default_reward("case-b", 0.8))
+ _write_complete_job_result(job_dir, ["case-a", physical_trial_root])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-a", "case-b"])
+
+ assert result["execution_status"] == "succeeded"
+ trials_dir = tmp_path / "results" / "opencode" / "with-skill" / "trials"
+ reward_paths = sorted(trials_dir.glob("*/reward.json"))
+ persisted_rewards = [json.loads(path.read_text(encoding="utf-8")) for path in reward_paths]
+ assert len(persisted_rewards) == 2
+ assert len({path.parent.name.rstrip(" .").casefold() for path in reward_paths}) == 2
+ assert {reward["entry_id"] for reward in persisted_rewards} == {"case-a", "case-b"}
+ assert {reward["trial_id"] for reward in persisted_rewards} == {"case-a", physical_trial_root}
+
+ loaded_agent = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ assert loaded_agent["rewards_complete"] is True
+ assert len(loaded_agent["rewards"]) == 2
+
+
+def test_persisted_trial_alias_keeps_exact_source_identity_with_trailing_space(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs"
+ rewards: list[dict[str, object]] = []
+ for index, trial_root_name in enumerate(("case", "case "), start=1):
+ agent_dir = job_dir / trial_root_name / "agent"
+ agent_dir.mkdir(parents=True)
+ trajectory = {
+ "schema_version": "ATIF-v1.2",
+ "agent": {"name": "opencode", "version": "test"},
+ "steps": [
+ {
+ "step_id": 1,
+ "source": "agent",
+ "message": f"source-{index}",
+ "tool_calls": [],
+ "observation": {"results": []},
+ }
+ ],
+ }
+ (agent_dir / "trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+ rewards.append(
+ {
+ **_default_reward(f"case-{index}", 1.0),
+ "_trial_name": trial_root_name,
+ "_trial_root_name": trial_root_name,
+ }
+ )
+
+ trials_dir = tmp_path / "results" / "trials"
+ collector_module._save_trials(
+ rewards,
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ persisted = [
+ (
+ json.loads((path / "reward.json").read_text(encoding="utf-8")),
+ json.loads((path / "trajectory.json").read_text(encoding="utf-8")),
+ )
+ for path in sorted(trials_dir.iterdir())
+ ]
+ assert {reward["trial_id"] for reward, _trajectory in persisted} == {"case", "case "}
+ assert {trajectory["steps"][0]["message"] for _reward, trajectory in persisted} == {
+ "source-1",
+ "source-2",
+ }
+
+
+def test_scored_and_unscored_portable_name_collision_persists_distinct_trials(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs"
+ scored_source = job_dir / "foo "
+ scored_source.mkdir(parents=True)
+ unscored_source = job_dir / "foo"
+ unscored_source.mkdir(parents=True)
+ (unscored_source / "exception.txt").write_text("unscored-source", encoding="utf-8")
+ rewards = [
+ {
+ **_default_reward("case-scored", 1.0),
+ "_trial_name": "foo ",
+ "_trial_root_name": "foo ",
+ }
+ ]
+ trials_dir = tmp_path / "results" / "trials"
+
+ collector_module._save_trials(
+ rewards,
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ trial_dirs = sorted(path for path in trials_dir.iterdir() if path.is_dir())
+ assert len(trial_dirs) == 2
+ assert len({path.name.rstrip(" .").casefold() for path in trial_dirs}) == 2
+ [scored_out] = [path for path in trial_dirs if (path / "reward.json").exists()]
+ [unscored_out] = [path for path in trial_dirs if (path / "failure.json").exists()]
+ assert json.loads((scored_out / "reward.json").read_text(encoding="utf-8"))["entry_id"] == "case-scored"
+ assert json.loads((unscored_out / "failure.json").read_text(encoding="utf-8"))["trial"] == "foo"
+ assert (unscored_out / "exception.txt").read_text(encoding="utf-8") == "unscored-source"
+
+
+def test_unscored_failure_metadata_is_bounded_and_report_readable(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs"
+ trial_source = job_dir / "unscored-trial"
+ trial_source.mkdir(parents=True)
+ private_tail = "private-task-name-tail"
+ (trial_source / "result.json").write_text(
+ json.dumps(
+ {
+ "task_name": ("x" * (3 * 1024 * 1024)) + private_tail,
+ "exception_info": {
+ "exception_type": "HarborTrialError",
+ "exception_message": "the trial did not produce a reward",
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+ trials_dir = tmp_path / "results" / "trials"
+
+ collector_module._save_trials(
+ [],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ failure_path = trials_dir / "unscored-trial" / "failure.json"
+ assert failure_path.stat().st_size <= 2 * 1024 * 1024
+ failure = json.loads(failure_path.read_text(encoding="utf-8"))
+ assert failure["status"] == "unscored"
+ assert len(failure["task_name"]) <= collector_module.REWARD_METADATA_TEXT_MAX_CHARS
+ assert failure["task_name"].endswith("...")
+ assert private_tail not in failure_path.read_text(encoding="utf-8")
+ diagnostics: list[dict[str, object]] = []
+ assert report_data._load_bounded_json(failure_path, diagnostics, artifact="failure") == failure
+ assert diagnostics == []
+
+
+@pytest.mark.skipif(os.name == "nt", reason="colon and backslash are not literal Windows child names")
+@pytest.mark.parametrize("trial_root_name", [r"foo\bar", "foo:bar"])
+def test_persisted_trial_alias_keeps_exact_posix_source_name(
+ tmp_path: Path,
+ trial_root_name: str,
+) -> None:
+ job_dir = tmp_path / "jobs"
+ trial_source = job_dir / trial_root_name
+ trial_source.mkdir(parents=True)
+ (trial_source / "trial.log").write_text("actual-source", encoding="utf-8")
+ trials_dir = tmp_path / "results" / "trials"
+
+ collector_module._save_trials(
+ [
+ {
+ **_default_reward("case-scored", 1.0),
+ "_trial_name": trial_root_name,
+ "_trial_root_name": trial_root_name,
+ }
+ ],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ [trial_out] = list(trials_dir.iterdir())
+ assert (trial_out / "trial.log").read_text(encoding="utf-8") == "actual-source"
+ assert json.loads((trial_out / "reward.json").read_text(encoding="utf-8"))["trial_id"] == trial_root_name
+
+
+def test_long_multistep_output_name_uses_portable_collision_alias(tmp_path: Path) -> None:
+ trial_root_name = "r" * 200
+ step_name = "s" * 100
+ job_dir = tmp_path / "jobs"
+ (job_dir / trial_root_name).mkdir(parents=True)
+ trials_dir = tmp_path / "results" / "trials"
+
+ collector_module._save_trials(
+ [
+ {
+ **_default_reward("case-scored", 1.0),
+ "_trial_name": trial_root_name,
+ "_trial_root_name": trial_root_name,
+ "_step_name": step_name,
+ }
+ ],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ [trial_out] = list(trials_dir.iterdir())
+ assert trial_out.name.startswith("skillevaluator-trial-collision-")
+ assert len(trial_out.name.encode("utf-8")) <= 240
+ assert (trial_out / "reward.json").is_file()
+
+
+@pytest.mark.parametrize("reserved_name", ["CON", "nul.txt", "COM1"])
+def test_windows_reserved_trial_root_uses_portable_collision_alias(reserved_name: str) -> None:
+ [(output_name, source_name)] = collector_module._persisted_trial_names(
+ [{"_trial_root_name": reserved_name}],
+ None,
+ )
+
+ assert output_name.startswith("skillevaluator-trial-collision-")
+ assert source_name == reserved_name
+
+
+@pytest.mark.parametrize("trailing_dot_name", ["foo.", "foo.."])
+def test_windows_trailing_dot_trial_root_uses_portable_collision_alias(trailing_dot_name: str) -> None:
+ [(output_name, source_name)] = collector_module._persisted_trial_names(
+ [{"_trial_root_name": trailing_dot_name}],
+ None,
+ )
+
+ assert output_name.startswith("skillevaluator-trial-collision-")
+ assert source_name == trailing_dot_name
+
+
+def test_saved_reward_drops_credential_shaped_custom_metric_key(tmp_path: Path) -> None:
+ credential = "sk-abcdefghijk"
+ job_dir = tmp_path / "jobs"
+ (job_dir / "case-001").mkdir(parents=True)
+ trials_dir = tmp_path / "results" / "trials"
+ reward = {
+ **_default_reward("case-001", 1.0),
+ "custom_metrics": {credential: 0.5, "quality": 0.7},
+ "_trial_name": "case-001",
+ "_trial_root_name": "case-001",
+ }
+
+ collector_module._save_trials(
+ [reward],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ persisted_text = (trials_dir / "case-001" / "reward.json").read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert credential not in persisted_text
+ assert persisted["custom_metrics"] == {"quality": 0.7}
+
+
+@pytest.mark.parametrize("surface", ["top_level", "custom_metrics", "metrics"])
+def test_saved_reward_preserves_allowlisted_sensitive_custom_metrics_and_safe_details(
+ tmp_path: Path,
+ surface: str,
+) -> None:
+ job_dir = tmp_path / "jobs"
+ (job_dir / "case-001").mkdir(parents=True)
+ trials_dir = tmp_path / "results" / "trials"
+ scores = {"secret_handling": 0.9, "auth_quality": 0.8, "token_efficiency": 0.7}
+ reward: dict[str, object] = {
+ "entry_id": "case-001",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 0.8,
+ "custom_details": {
+ name: {
+ "reason": f"Evidence retained for {name}",
+ "api_key": "sk-abcdefghijk",
+ }
+ for name in scores
+ },
+ "_trial_name": "case-001",
+ "_trial_root_name": "case-001",
+ }
+ if surface == "top_level":
+ reward.update(scores)
+ elif surface == "custom_metrics":
+ reward["custom_metrics"] = scores
+ else:
+ reward["metrics"] = {name: {"score": score} for name, score in scores.items()}
+
+ collector_module._save_trials(
+ [reward],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ persisted_text = (trials_dir / "case-001" / "reward.json").read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert "sk-abcdefghijk" not in persisted_text
+ assert extract_custom_metrics(persisted) == scores
+ for name in scores:
+ assert persisted["custom_details"][name] == {
+ "reason": f"Evidence retained for {name}",
+ "api_key": "",
+ }
+
+
+@pytest.mark.parametrize("surface", ["top_level", "custom_metrics", "metrics"])
+def test_saved_reward_omits_rejected_metric_names_from_ordinary_details(
+ tmp_path: Path,
+ surface: str,
+) -> None:
+ job_dir = tmp_path / "jobs"
+ (job_dir / "case-001").mkdir(parents=True)
+ trials_dir = tmp_path / "results" / "trials"
+ rejected_name = "api_key_quality"
+ reward: dict[str, object] = {
+ "entry_id": "case-001",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 0.8,
+ "details": {
+ rejected_name: {"reason": "must not become a redaction alias"},
+ "quality": {"reason": "bounded evidence"},
+ },
+ "_trial_name": "case-001",
+ "_trial_root_name": "case-001",
+ }
+ scores: dict[str, object] = {rejected_name: 0.9, "quality": 0.8}
+ if surface == "top_level":
+ reward.update(scores)
+ elif surface == "custom_metrics":
+ reward["custom_metrics"] = scores
+ else:
+ reward["metrics"] = {name: {"score": score} for name, score in scores.items()}
+
+ collector_module._save_trials(
+ [reward],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ persisted_text = (trials_dir / "case-001" / "reward.json").read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert rejected_name not in persisted_text
+ assert persisted["details"] == {"quality": {"reason": "bounded evidence"}}
+
+
+@pytest.mark.parametrize(
+ "rejected_value",
+ [{"reason": "nonnumeric"}, "nonnumeric", None],
+ ids=("dict", "string", "null"),
+)
+def test_saved_reward_omits_nonnumeric_unpublishable_top_level_and_detail_keys(
+ tmp_path: Path,
+ rejected_value: object,
+) -> None:
+ job_dir = tmp_path / "jobs"
+ (job_dir / "case-001").mkdir(parents=True)
+ trials_dir = tmp_path / "results" / "trials"
+ rejected_names = [
+ "api_key_quality",
+ "tokens_quality",
+ "quality_sk-abcdefghijk",
+ "x" * (MAX_CUSTOM_METRIC_NAME_BYTES + 1),
+ ]
+ reward: dict[str, object] = {
+ "entry_id": "case-001",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": 0.8,
+ "quality": 0.8,
+ "details": {
+ **dict.fromkeys(rejected_names, rejected_value),
+ "quality": {"reason": "bounded evidence"},
+ },
+ **dict.fromkeys(rejected_names, rejected_value),
+ "_trial_name": "case-001",
+ "_trial_root_name": "case-001",
+ }
+
+ collector_module._save_trials(
+ [reward],
+ trials_dir,
+ job_dir,
+ skill_name="demo",
+ agent="opencode",
+ variant="with_skill",
+ )
+
+ persisted_text = (trials_dir / "case-001" / "reward.json").read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert all(name not in persisted_text for name in rejected_names)
+ assert persisted["quality"] == 0.8
+ assert persisted["details"] == {"quality": {"reason": "bounded evidence"}}
+
+
+@pytest.mark.parametrize("surface", ["custom_metrics", "metrics", "top_level"])
+def test_credential_custom_metric_names_never_reach_aggregates_or_paired_artifacts(
+ tmp_path: Path,
+ surface: str,
+) -> None:
+ credentials = [
+ "sk-abcdefghijk",
+ "ghp_" + ("a" * 36),
+ "gho_" + ("a" * 36),
+ "ghu_" + ("a" * 36),
+ "ghs_" + ("a" * 36),
+ "ghr_" + ("a" * 36),
+ "ghs_123456789_" + ("a" * 32) + "." + ("b" * 32) + "." + ("c" * 32),
+ "github_pat_" + ("a" * 30),
+ "".join(("xoxb-", "1234567890-abcdefghijklmnopqrstuvwx")), # noqa: FLY002
+ "AIza" + ("A" * 35),
+ "glpat-" + ("a" * 20),
+ ]
+ for variant, quality in (("with", 0.8), ("without", 0.3)):
+ job_dir = tmp_path / "jobs" / f"demo-opencode-{variant}"
+ reward = _default_reward("case-001", quality)
+ custom = {**dict.fromkeys(credentials, 0.5), "quality": quality}
+ if surface == "top_level":
+ reward.update(custom)
+ else:
+ reward[surface] = custom
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
+
+ result = _collect(tmp_path, skip_baseline=False, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["custom_with_skill"] == {"quality": 0.8}
+ assert agent["custom_without_skill"] == {"quality": 0.3}
+ assert set(agent["custom_lift"]) == {"quality"}
+ assert all(credential not in json.dumps(result) for credential in credentials)
+ assert "" not in json.dumps(agent["custom_lift"])
+
+ for artifact in (tmp_path / "results").rglob("*.json"):
+ text = artifact.read_text(encoding="utf-8")
+ json.loads(text)
+ assert all(credential not in text for credential in credentials)
+ loaded = report_data.load_agent_data(tmp_path / "results")["opencode"]
+ assert loaded["rewards_complete"] is True
+
+
+@pytest.mark.parametrize("surface", ["custom_metrics", "metrics"])
+def test_invalid_claimed_custom_metric_values_are_omitted_without_score_drift(
+ tmp_path: Path,
+ surface: str,
+) -> None:
+ invalid_entries: dict[str, object] = {
+ "negative_scalar": -0.1,
+ "above_one_scalar": 1.1,
+ "string_scalar": "not-a-score",
+ "null_scalar": None,
+ "boolean_scalar": True,
+ "negative_dict": {"score": -0.1},
+ "above_one_dict": {"score": 1.1},
+ "string_dict": {"score": "not-a-score"},
+ "null_dict": {"score": None},
+ "boolean_dict": {"score": False},
+ }
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ reward = _default_reward("case-001", 1.0)
+ reward[surface] = {**invalid_entries, "quality": 0.8}
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ assert result["execution_status"] == "succeeded"
+ assert result["agents"]["opencode"]["custom_with_skill"] == {"quality": 0.8}
+ reward_path = next((tmp_path / "results/opencode/with-skill/trials").glob("*/reward.json"))
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert persisted[surface] == {"quality": 0.8}
+ for artifact in (tmp_path / "results").rglob("*.json"):
+ text = artifact.read_text(encoding="utf-8")
+ json.loads(text)
+ assert all(name not in text for name in invalid_entries)
+
+
+@pytest.mark.parametrize("surface", ["custom_metrics", "metrics"])
+def test_invalid_custom_metric_cardinality_is_stripped_before_publication(
+ tmp_path: Path,
+ surface: str,
+) -> None:
+ invalid_entries = {f"invalid_{index:03d}": None for index in range(MAX_CUSTOM_METRICS + 1)}
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ reward = _default_reward("case-001", 1.0)
+ reward[surface] = {**invalid_entries, "quality": 0.8}
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ assert result["execution_status"] == "succeeded"
+ assert result["agents"]["opencode"]["custom_with_skill"] == {"quality": 0.8}
+ summary_path = tmp_path / "results/opencode/with-skill/summary.json"
+ assert summary_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ reward_path = next((summary_path.parent / "trials").glob("*/reward.json"))
+ persisted_text = reward_path.read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert persisted[surface] == {"quality": 0.8}
+ assert all(name not in persisted_text for name in invalid_entries)
-def test_single_step_result_fallback_honors_failed_top_level_sidecar(tmp_path: Path) -> None:
+def test_custom_metric_cardinality_fails_before_aggregation_and_stays_report_readable(tmp_path: Path) -> None:
job_dir = tmp_path / "jobs" / "demo-opencode-with"
- trial_name = "case-001__attempt"
- trial_dir = job_dir / trial_name
- _numeric, rich = _failed_judge_artifacts("case-001")
- trial_dir.mkdir(parents=True)
- (trial_dir / "result.json").write_text(
- json.dumps(
- {
- "trial_name": trial_name,
- "task_name": "case-001",
- "verifier_result": {"rewards": _default_reward("case-001", 1.0)},
- }
- ),
- encoding="utf-8",
- )
- verifier_dir = trial_dir / "verifier"
- verifier_dir.mkdir()
- (verifier_dir / "skill_evaluator_reward.json").write_text(json.dumps(rich), encoding="utf-8")
- _write_complete_job_result(job_dir, [trial_name])
-
- [extracted] = _extract_rewards(job_dir)
- assert extracted["evaluation_status"] == "failed"
+ reward = _default_reward("case-001", 1.0)
+ reward["custom_metrics"] = {f"metric_{index:03d}": 1.0 for index in range(MAX_CUSTOM_METRICS + 1)}
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
assert result["execution_status"] == "failed"
- assert result["agents"]["opencode"]["with_skill"] == {}
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
+ assert agent["custom_with_skill"] == {}
+ summary_path = tmp_path / "results" / "opencode" / "with-skill" / "summary.json"
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
+ assert summary_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ assert summary["execution_status"] == "failed"
+ [reward_path] = (summary_path.parent / "trials").glob("*/reward.json")
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert persisted["evaluation_status"] == "failed"
+ assert "custom metric" in json.dumps(persisted["evaluation_errors"]).casefold()
+
+
+def test_condition_custom_metric_union_fails_before_summary_or_custom_lift(tmp_path: Path) -> None:
+ per_reward = MAX_CUSTOM_METRICS // 2 + 1
+ case_ids = ["case-a", "case-b"]
+ for variant in ("with", "without"):
+ job_dir = tmp_path / "jobs" / f"demo-opencode-{variant}"
+ for case_index, case_id in enumerate(case_ids):
+ reward = _default_reward(case_id, 1.0)
+ reward["custom_metrics"] = {f"{variant}_{case_index}_{index:03d}": 1.0 for index in range(per_reward)}
+ _write_reward(job_dir, f"{case_id}__attempt", reward)
+ _write_complete_job_result(job_dir, [f"{case_id}__attempt" for case_id in case_ids])
+ result = _collect(tmp_path, skip_baseline=False, case_ids=case_ids)
-def test_failed_sidecar_overrides_conflicting_reward_status_and_errors(tmp_path: Path) -> None:
- job_dir = tmp_path / "jobs" / "demo-opencode-with"
- trial_name = "case-001__attempt"
- secret = "sk-authoritativesidecar123"
- reward = {
- **_default_reward("case-001", 1.0),
- "evaluation_status": "succeeded",
- "evaluation_errors": {"accuracy": "stale reward error"},
- }
- _numeric, rich = _failed_judge_artifacts("case-001", reason=f"provider echoed {secret}")
- rich["evaluation_status"] = "error"
- _write_reward(job_dir, trial_name, reward, sidecar=rich)
- _write_complete_job_result(job_dir, [trial_name])
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "failed"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
+ assert agent["conditions"]["without_skill"]["scored_attempts"] == 0
+ assert agent["custom_with_skill"] == {}
+ assert agent["custom_without_skill"] == {}
+ assert agent["custom_lift"] == {}
+ assert not (tmp_path / "results" / "opencode" / "custom_lift.json").exists()
+ for summary_path in (tmp_path / "results" / "opencode").glob("*/summary.json"):
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
+ assert summary_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ assert summary["execution_status"] == "failed"
- [extracted] = _extract_rewards(job_dir)
- assert extracted["evaluation_status"] == "failed"
- assert extracted["evaluation_errors"]["accuracy"] == "provider echoed sk-"
+
+def test_oversized_custom_metric_name_is_rejected_without_published_alias(tmp_path: Path) -> None:
+ oversized_name = "x" * (MAX_CUSTOM_METRIC_NAME_BYTES + 1)
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ reward = _default_reward("case-001", 1.0)
+ reward["custom_metrics"] = {oversized_name: 1.0, "quality": 0.8}
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
assert result["execution_status"] == "failed"
- assert secret not in " ".join(result["execution_errors"])
+ assert result["agents"]["opencode"]["custom_with_skill"] == {}
+ for artifact in (tmp_path / "results").rglob("*.json"):
+ text = artifact.read_text(encoding="utf-8")
+ json.loads(text)
+ assert oversized_name not in text
-def test_default_plus_custom_cannot_rescue_an_incomplete_default_reward(tmp_path: Path) -> None:
+def test_oversized_custom_metric_in_custom_sidecar_fails_standard_reward(tmp_path: Path) -> None:
+ oversized_name = "x" * (MAX_CUSTOM_METRIC_NAME_BYTES + 1)
job_dir = tmp_path / "jobs" / "demo-opencode-with"
- trial_name = "case-001__attempt"
- numeric, rich = _failed_judge_artifacts("case-001")
_write_reward(
job_dir,
- trial_name,
- numeric,
- sidecar=rich,
- custom={"overall": 1.0, "domain_quality": 1.0, "details": {"domain_quality": {"reason": "perfect"}}},
+ "case-001__attempt",
+ _default_reward("case-001", 1.0),
+ custom={"custom_metrics": {oversized_name: 0.5}},
)
- _write_complete_job_result(job_dir, [trial_name])
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
agent = result["agents"]["opencode"]
assert result["execution_status"] == "failed"
- assert agent["with_skill"] == {}
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 0
assert agent["custom_with_skill"] == {}
- assert agent["pass_at_k"]["with_skill"] == {}
+ [reward_path] = (tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json")
+ persisted_text = reward_path.read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert oversized_name not in persisted_text
+ assert persisted["evaluation_status"] == "failed"
+ assert "custom metric" in json.dumps(persisted["evaluation_errors"]).casefold()
-def test_custom_only_overall_reward_remains_scoreable(tmp_path: Path) -> None:
+def test_custom_sidecar_details_are_limited_to_numeric_custom_metric_names(tmp_path: Path) -> None:
job_dir = tmp_path / "jobs" / "demo-opencode-with"
- trial_name = "case-001__attempt"
+ custom_details = {f"detail_only_{index:05d}": "unused" for index in range(10_000)}
+ custom_details.update(
+ {
+ "quality": {"reason": "bounded evidence"},
+ "api_key_quality": {"reason": "must not become a redaction alias"},
+ }
+ )
_write_reward(
job_dir,
- trial_name,
- {"entry_id": "case-001", "overall": 0.75, "domain_quality": 0.9, "token_efficiency": 0.8},
+ "case-001__attempt",
+ _default_reward("case-001", 1.0),
+ custom={
+ "custom_metrics": {"quality": 0.8, "api_key_quality": 0.9},
+ "details": custom_details,
+ },
)
- _write_complete_job_result(job_dir, [trial_name])
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
- agent = result["agents"]["opencode"]
assert result["execution_status"] == "succeeded"
- assert agent["with_skill"] == {}
- assert agent["custom_with_skill"] == {"domain_quality": 0.9, "token_efficiency": 0.8}
- assert agent["pass_at_k"]["with_skill"]["rate"] == 1.0
- assert agent["conditions"]["without_skill"]["execution_status"] == "skipped"
- persisted = json.loads(
- (tmp_path / "results" / "opencode" / "with-skill" / "trials" / trial_name / "reward.json").read_text(
- encoding="utf-8"
- )
+ assert result["agents"]["opencode"]["custom_with_skill"] == {"quality": 0.8}
+ reward_path = next((tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json"))
+ persisted_text = reward_path.read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert persisted["custom_details"] == {"quality": {"reason": "bounded evidence"}}
+ assert "detail_only_" not in persisted_text
+ assert "api_key_quality" not in persisted_text
+
+ findings = report._extract_findings(
+ [persisted],
+ canonical_scores={"quality": 0.8},
)
- assert persisted["token_efficiency"] == 0.8
+ finding_metrics = {finding["metric"] for finding in findings}
+ assert "quality" in finding_metrics
+ assert not any(metric.startswith("detail_only_") for metric in finding_metrics)
+ assert "api_key_quality" not in finding_metrics
-def test_mixed_failed_condition_suppresses_published_quality_and_paired_artifacts(tmp_path: Path) -> None:
- jobs_dir = tmp_path / "jobs"
- with_job = jobs_dir / "demo-opencode-with"
- without_job = jobs_dir / "demo-opencode-without"
+def test_safe_sensitive_custom_metric_details_survive_with_nested_secrets_redacted(
+ tmp_path: Path,
+) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ credential = "sk-abcdefghijk"
+ _write_reward(
+ job_dir,
+ "case-001__attempt",
+ _default_reward("case-001", 1.0),
+ custom={
+ "custom_metrics": {"secret_handling": 0.9, "token_efficiency": 0.8},
+ "details": {
+ "secret_handling": {
+ "reason": f"Handled {credential} without disclosure",
+ "api_key": credential,
+ },
+ "token_efficiency": {
+ "reason": "Used the token budget efficiently",
+ "authorization": "Bearer abcdefghijklmnop",
+ },
+ },
+ },
+ )
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
- _write_reward(with_job, "case-001__attempt", _default_reward("case-001", 0.9))
- numeric, rich = _failed_judge_artifacts("case-002")
- _write_reward(with_job, "case-002__attempt", numeric, sidecar=rich)
- _write_complete_job_result(with_job, ["case-001__attempt", "case-002__attempt"])
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
- for case_id in ("case-001", "case-002"):
- _write_reward(without_job, f"{case_id}__attempt", _default_reward(case_id, 0.2))
- _write_complete_job_result(without_job, ["case-001__attempt", "case-002__attempt"])
+ assert result["execution_status"] == "succeeded"
+ reward_path = next((tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json"))
+ persisted_text = reward_path.read_text(encoding="utf-8")
+ persisted = json.loads(persisted_text)
+ assert credential not in persisted_text
+ assert persisted["custom_details"]["secret_handling"] == {
+ "reason": "Handled sk- without disclosure",
+ "api_key": "",
+ }
+ assert persisted["custom_details"]["token_efficiency"] == {
+ "reason": "Used the token budget efficiently",
+ "authorization": "",
+ }
- result = _collect(tmp_path, skip_baseline=False, case_ids=["case-001", "case-002"])
+ findings = report._extract_findings(
+ [persisted],
+ canonical_scores={"secret_handling": 0.9, "token_efficiency": 0.8},
+ )
+ findings_by_metric = {finding["metric"]: finding for finding in findings}
+ assert "Handled sk- without disclosure" in findings_by_metric["secret_handling"]["reasons"], findings
+ assert "Used the token budget efficiently" in findings_by_metric["token_efficiency"]["reasons"], findings
+
+
+def test_unpublishable_detail_key_amplification_is_removed_before_scoring(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ reward = _default_reward("case-001", 1.0)
+ reward["details"] = {f"api_key_{index:020d}": "" for index in range(49_000)}
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
agent = result["agents"]["opencode"]
- assert result["execution_status"] == "failed"
- assert agent["conditions"]["with_skill"]["execution_status"] == "failed"
+ assert result["execution_status"] == "succeeded"
assert agent["conditions"]["with_skill"]["scored_attempts"] == 1
- assert agent["conditions"]["without_skill"]["execution_status"] == "succeeded"
- assert agent["with_skill"] == {}
- assert agent["custom_with_skill"] == {}
- assert agent["dimensions_with_skill"] == {}
- assert agent["pass_at_k"]["with_skill"] == {}
- assert agent["without_skill"] == dict.fromkeys(DEFAULT_METRICS, 0.2)
- assert agent["dimensions_without_skill"]
- assert agent["pass_at_k"]["without_skill"]["rate"] == 0.0
- assert agent["lift"] == {}
- assert agent["custom_lift"] == {}
- assert agent["pass_at_k"]["lift"] == {}
- assert agent["security_attribution"] == {}
+ reward_path = next((tmp_path / "results" / "opencode" / "with-skill" / "trials").glob("*/reward.json"))
+ assert reward_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert persisted["details"] == {}
+ assert "api_key_" not in reward_path.read_text(encoding="utf-8")
- results_dir = tmp_path / "results" / "opencode"
- persisted = json.loads((results_dir / "with-skill" / "summary.json").read_text(encoding="utf-8"))
- assert persisted["scores"] == {}
- assert persisted["custom_scores"] == {}
- assert persisted["dimensions"] == {}
- assert persisted["pass_at_k"] == {}
- assert not (results_dir / "lift.json").exists()
- assert not (results_dir / "custom_lift.json").exists()
- assert not (results_dir / "pass_at_k_lift.json").exists()
- assert not (results_dir / "security_attribution.json").exists()
- # Both source artifacts remain available as redacted trial diagnostics.
- assert (results_dir / "with-skill" / "trials" / "case-001__attempt" / "reward.json").exists()
- failed_reward = results_dir / "with-skill" / "trials" / "case-002__attempt" / "reward.json"
- assert json.loads(failed_reward.read_text(encoding="utf-8"))["evaluation_status"] == "failed"
- skill_dir = tmp_path / "demo"
- skill_dir.mkdir()
- html = render_agent_eval_html_report(skill_dir, tmp_path / "results", use_llm_judge=False).read_text(
- encoding="utf-8"
- )
- assert "Evaluation incomplete" in html
- assert "Required judge evaluation failed" in html
- assert re.findall(r'class="t3-dim-score"[^>]*>([^<]+)', html) == ["N/A"] * 5
- assert "return raw === null || raw === undefined ? null : Number(raw);" in html
- payload_match = re.search(
- r'',
- html,
- re.DOTALL,
+def test_security_attribution_cannot_make_persisted_reward_disagree_with_score(tmp_path: Path) -> None:
+ job_dir = tmp_path / "jobs" / "demo-opencode-with"
+ reward = _default_reward("case-001", 1.0)
+ reward["details"] = {
+ "security": {
+ "findings": [{"score_impact": True} for _ in range(10_000)],
+ }
+ }
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
+
+ result = _collect(tmp_path, skip_baseline=True, case_ids=["case-001"])
+
+ agent = result["agents"]["opencode"]
+ assert result["execution_status"] == "succeeded"
+ assert agent["conditions"]["with_skill"]["scored_attempts"] == 1
+ summary_path = tmp_path / "results" / "opencode" / "with-skill" / "summary.json"
+ summary = json.loads(summary_path.read_text(encoding="utf-8"))
+ assert summary["execution_status"] == "succeeded"
+ assert summary["scored_attempts"] == 1
+ reward_path = next((summary_path.parent / "trials").glob("*/reward.json"))
+ persisted = json.loads(reward_path.read_text(encoding="utf-8"))
+ assert reward_path.stat().st_size <= report_data._MAX_JSON_BYTES
+ assert persisted.get("evaluation_status") != "failed"
+ assert all(persisted[metric] == 1.0 for metric in DEFAULT_METRICS)
+
+
+def test_security_improvement_projection_keeps_paired_artifacts_scoreable_and_bounded(tmp_path: Path) -> None:
+ padding = "x" * 1_500_000
+ with_reward = _default_reward("case-001", 1.0)
+ with_reward["details"] = {
+ "padding": padding,
+ "security": {"score": 1.0, "findings": []},
+ }
+ baseline_reward = _default_reward("case-001", 1.0)
+ baseline_reward["details"] = {
+ "security": {
+ "score": 0.0,
+ "findings": [
+ {
+ "type": "unsafe_action",
+ "message": padding,
+ "score_impact": True,
+ }
+ ],
+ }
+ }
+ for variant, reward in (("with", with_reward), ("without", baseline_reward)):
+ job_dir = tmp_path / "jobs" / f"demo-opencode-{variant}"
+ _write_reward(job_dir, "case-001__attempt", reward)
+ _write_complete_job_result(job_dir, ["case-001__attempt"])
+
+ result = _collect(tmp_path, skip_baseline=False, case_ids=["case-001"])
+
+ assert result["execution_status"] == "succeeded"
+ with_summary = json.loads((tmp_path / "results/opencode/with-skill/summary.json").read_text(encoding="utf-8"))
+ assert with_summary["execution_status"] == "succeeded"
+ with_reward_path = next((tmp_path / "results/opencode/with-skill/trials").glob("*/reward.json"))
+ persisted = json.loads(with_reward_path.read_text(encoding="utf-8"))
+ derived = persisted["details"]["security"]["findings"][0]
+ assert persisted.get("evaluation_status") != "failed"
+ assert derived["type"] == "skill_reduced_unsafe_behavior"
+ assert derived["evidence"].startswith("Without-skill baseline contained 1")
+ assert len(derived["evidence"]) < 256
+ assert with_reward_path.stat().st_size <= report_data._MAX_JSON_BYTES
+
+
+def test_duplicate_physical_reward_rows_are_preserved_and_fail_closed() -> None:
+ reward = {
+ **_default_reward("case-a", 0.5),
+ "_trial_name": "case-a__attempt",
+ "_trial_root_name": "case-a__attempt",
+ }
+ rewards = [dict(reward), dict(reward)]
+
+ persisted_names = collector_module._persisted_trial_names(rewards, None)
+ assert len({name for name, _trial_root in persisted_names}) == 2
+ assert {trial_root for _name, trial_root in persisted_names} == {"case-a__attempt"}
+
+ execution = collector_module._condition_execution_summary(
+ rewards,
+ expected_case_ids=["case-a"],
+ expected_cases=1,
+ n_attempts=1,
+ job_failure="",
)
- assert payload_match is not None
- report_payload = json.loads(payload_match.group(1))
- for dimension in report_payload["agents"]["opencode"]["dimensions"]:
- assert dimension["with_skill"] is None
- assert dimension["score"] is None
- assert dimension["verdict"] is None
- assert "0.00" not in str(dimension["explanation"])
- assert all("0.00" not in bullet for bullet in dimension["reasoning_bullets"])
+ assert execution["execution_status"] == "failed"
+ assert "duplicate reward rows" in " ".join(execution["execution_errors"])
+
+
+def test_collision_names_do_not_encode_rejected_raw_identifiers() -> None:
+ first_marker = "synthetic-private-one"
+ second_marker = "synthetic-private-two"
+ rewards = [
+ {"_trial_root_name": f"token={first_marker}:invalid"},
+ {"_trial_root_name": f"token={second_marker}:invalid"},
+ ]
+
+ persisted_names = collector_module._persisted_trial_names(rewards, None)
+
+ assert persisted_names == [
+ ("skillevaluator-trial-collision-000001", f"token={first_marker}:invalid"),
+ ("skillevaluator-trial-collision-000002", f"token={second_marker}:invalid"),
+ ]
+ rendered_names = " ".join(name for name, _trial_root in persisted_names)
+ assert first_marker not in rendered_names
+ assert second_marker not in rendered_names
+ assert collector_module._persisted_trial_names(rewards, None) == persisted_names
+ assert collector_module._persisted_trial_names([{"_trial_root_name": "case-a", "_step_name": "x"}], None) == [
+ ("case-a__x", "case-a")
+ ]
def test_reused_results_remove_stale_generated_quality_but_preserve_unrelated_files(
@@ -2384,6 +4011,83 @@ def capture_suggestions(*_args, **_kwargs):
assert not (agent_dir / "findings.json").exists()
+@pytest.mark.parametrize(
+ ("agent_specs", "expected_best"),
+ [
+ ([("standard", "standard", 0.8), ("custom", "custom", 0.9)], "custom"),
+ ([("custom-low", "custom", 0.6), ("custom-high", "custom", 0.9)], "custom-high"),
+ ],
+ ids=("standard-and-custom", "custom-only"),
+)
+def test_findings_multi_agent_ranks_persisted_condition_overall_across_contracts(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ agent_specs: list[tuple[str, str, float]],
+ expected_best: str,
+) -> None:
+ agents: dict[str, dict[str, object]] = {}
+ for agent, contract, score in agent_specs:
+ condition_dir = tmp_path / agent / "with-skill"
+ trial_dir = condition_dir / "trials" / "case-001__attempt"
+ trial_dir.mkdir(parents=True)
+ if contract == "standard":
+ scores = dict.fromkeys(DEFAULT_METRICS, score)
+ custom_scores: dict[str, float] = {}
+ metrics = list(DEFAULT_METRICS)
+ reward: dict[str, object] = _default_reward("case-001", score)
+ else:
+ scores = {}
+ custom_scores = {"domain_quality": score}
+ metrics = []
+ reward = {
+ "entry_id": "case-001",
+ "metric_set": CUSTOM_ONLY_METRIC_SET,
+ "overall": score,
+ "custom_metrics": {"domain_quality": score},
+ "custom_details": {"domain_quality": {"reason": "contract-grounded evidence"}},
+ }
+ (condition_dir / "summary.json").write_text(
+ json.dumps(
+ {
+ "agent": agent,
+ "scores": scores,
+ "custom_scores": custom_scores,
+ "overall_score": score,
+ "metrics": metrics,
+ "execution_status": "succeeded",
+ "execution_errors": [],
+ "expected_attempts": 1,
+ "scored_attempts": 1,
+ "num_trials": 1,
+ "num_reward_rows": 1,
+ }
+ ),
+ encoding="utf-8",
+ )
+ (trial_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")
+ agents[agent] = {
+ "execution_status": "succeeded",
+ "with_skill": scores,
+ "custom_with_skill": custom_scores,
+ "conditions": {"with_skill": {"execution_status": "succeeded"}},
+ }
+
+ monkeypatch.setattr(report, "_generate_suggestions_structured", lambda *_args, **_kwargs: [])
+
+ rendered = report.display_findings_report(
+ {"agents": agents},
+ "demo",
+ [agent for agent, _contract, _score in agent_specs],
+ tmp_path,
+ )
+
+ assert rendered
+ for agent, _contract, _score in agent_specs:
+ artifact = json.loads((tmp_path / agent / "findings.json").read_text(encoding="utf-8"))
+ expected_mode = "passing_next_steps" if agent == expected_best else "not_generated"
+ assert artifact["suggestion_mode"] == expected_mode
+
+
def test_findings_multi_agent_selects_and_writes_only_successful_agent(tmp_path: Path, monkeypatch) -> None:
for agent, score in (("failed", 1.0), ("succeeded", 0.8)):
condition_dir = tmp_path / agent / "with-skill"
diff --git a/tests/test_oss_packaging.py b/tests/test_oss_packaging.py
index 5d80f381..689267d7 100644
--- a/tests/test_oss_packaging.py
+++ b/tests/test_oss_packaging.py
@@ -5,12 +5,15 @@
from __future__ import annotations
+import ast
+import importlib.util
import re
import subprocess
import sys
import tarfile
import tomllib
import zipfile
+from importlib.metadata import metadata
from pathlib import Path
from click.testing import CliRunner
@@ -19,6 +22,11 @@
from packaging.version import Version
from skillevaluator.cli import cli
+from skillevaluator.tier3_environments import (
+ HARBOR_ENVIRONMENT_EXTRAS,
+ HARBOR_ENVIRONMENTS,
+ HARBOR_NATIVE_ENV_MODES,
+)
REPO_ROOT = Path(__file__).resolve().parents[1]
PUBLIC_REQUIRED_FILES = (
@@ -119,6 +127,219 @@ def test_public_extras_use_public_dependency_sources() -> None:
)
+def test_harbor_022_dependency_contract_keeps_base_install_isolated() -> None:
+ project = _project()
+ extras = project["project"]["optional-dependencies"]
+ base_requirements = [Requirement(raw) for raw in project["project"]["dependencies"]]
+ llm_requirements = [Requirement(raw) for raw in extras["llm"]]
+ tier3_requirements = [Requirement(raw) for raw in extras["tier3"]]
+
+ base_names = {canonicalize_name(requirement.name) for requirement in base_requirements}
+ harbor_requirements = [
+ requirement for requirement in tier3_requirements if canonicalize_name(requirement.name) == "harbor"
+ ]
+ litellm_requirements = [
+ requirement for requirement in llm_requirements if canonicalize_name(requirement.name) == "litellm"
+ ]
+
+ assert base_names.isdisjoint({"harbor", "litellm"})
+ assert len(harbor_requirements) == 1
+ assert not harbor_requirements[0].extras
+ assert str(harbor_requirements[0].specifier) == "==0.22.0"
+ assert len(litellm_requirements) == 1
+ litellm_specifier = litellm_requirements[0].specifier
+ assert litellm_specifier.contains(Version("1.92.0"), prereleases=True)
+ assert litellm_specifier.contains(Version("1.93.0"), prereleases=True)
+ assert not litellm_specifier.contains(Version("1.94.0.dev0"), prereleases=True)
+ declared_names = base_names | {
+ canonicalize_name(Requirement(raw).name) for requirements in extras.values() for raw in requirements
+ }
+ assert "claude-agent-sdk" not in declared_names
+
+ lock = _lock()
+ locked_versions = {
+ name: [Version(package["version"]) for package in lock["package"] if package["name"] == name]
+ for name in ("harbor", "litellm")
+ }
+ assert locked_versions["harbor"] == [Version("0.22.0")]
+ assert len(locked_versions["litellm"]) == 1
+ assert litellm_specifier.contains(locked_versions["litellm"][0], prereleases=True)
+
+ root_lock = next(package for package in lock["package"] if package["name"] == "skillevaluator")
+ locked_requirements = root_lock["metadata"]["requires-dist"]
+ locked_harbor = [requirement for requirement in locked_requirements if requirement["name"] == "harbor"]
+ locked_litellm = [requirement for requirement in locked_requirements if requirement["name"] == "litellm"]
+ assert [requirement["specifier"] for requirement in locked_harbor] == ["==0.22.0"]
+ assert [requirement["specifier"] for requirement in locked_litellm] == [">=1.92.0,<1.94.0.dev0"]
+
+
+def test_harbor_environment_extra_mapping_matches_installed_metadata() -> None:
+ provided_extras = set(metadata("harbor").get_all("Provides-Extra") or ())
+ system_or_base_backends = {"docker", "openshift", "apple-container", "singularity"}
+
+ assert len(HARBOR_ENVIRONMENTS) == 27
+ assert len(HARBOR_NATIVE_ENV_MODES) == 26
+ assert frozenset(HARBOR_ENVIRONMENTS) - {"local"} == HARBOR_NATIVE_ENV_MODES
+ assert set(HARBOR_ENVIRONMENT_EXTRAS) == HARBOR_NATIVE_ENV_MODES
+ assert {mode for mode, extra in HARBOR_ENVIRONMENT_EXTRAS.items() if extra is None} == system_or_base_backends
+ assert {extra for extra in HARBOR_ENVIRONMENT_EXTRAS.values() if extra is not None} <= provided_extras
+ assert {
+ mode: extra for mode, extra in HARBOR_ENVIRONMENT_EXTRAS.items() if extra is not None and extra != mode
+ } == {"ack": "gke", "cua-cloud": "cua"}
+ assert HARBOR_ENVIRONMENT_EXTRAS["ack"] == "gke"
+ assert "cloud" in provided_extras
+ assert HARBOR_ENVIRONMENT_EXTRAS["cua-cloud"] == "cua"
+
+
+def _harbor_022_environment_kwargs_from_installed_source() -> dict[str, frozenset[str]]:
+ """Read the pinned Harbor sources without importing optional backend SDKs."""
+ harbor_spec = importlib.util.find_spec("harbor")
+ assert harbor_spec is not None and harbor_spec.origin is not None
+ environments_root = Path(harbor_spec.origin).parent / "environments"
+
+ source_trees: dict[Path, ast.Module] = {
+ path: ast.parse(path.read_text(encoding="utf-8")) for path in environments_root.rglob("*.py")
+ }
+ class_index: dict[str, tuple[ast.ClassDef, ast.Module]] = {}
+ for tree in source_trees.values():
+ for node in tree.body:
+ if isinstance(node, ast.ClassDef):
+ class_index[node.name] = (node, tree)
+
+ def literal_kwarg_accesses(node: ast.AST, *, name: str | None = None, attribute: str | None = None) -> set[str]:
+ keys: set[str] = set()
+ for candidate in ast.walk(node):
+ source: ast.AST | None = None
+ if isinstance(candidate, ast.Call) and isinstance(candidate.func, ast.Attribute):
+ if candidate.func.attr in {"get", "pop"} and candidate.args:
+ source = candidate.func.value
+ key_node = candidate.args[0]
+ else:
+ continue
+ elif isinstance(candidate, ast.Subscript):
+ source = candidate.value
+ key_node = candidate.slice
+ else:
+ continue
+ source_matches = (name is not None and isinstance(source, ast.Name) and source.id == name) or (
+ attribute is not None and isinstance(source, ast.Attribute) and source.attr == attribute
+ )
+ if source_matches and isinstance(key_node, ast.Constant) and isinstance(key_node.value, str):
+ keys.add(key_node.value)
+ return keys
+
+ resolved_classes: dict[str, frozenset[str]] = {}
+
+ def class_kwargs(class_name: str) -> frozenset[str]:
+ if class_name in resolved_classes:
+ return resolved_classes[class_name]
+ class_node, module_tree = class_index[class_name]
+ names: set[str] = set()
+ for base in class_node.bases:
+ base_name = (
+ base.id if isinstance(base, ast.Name) else base.attr if isinstance(base, ast.Attribute) else None
+ )
+ if base_name in class_index:
+ names.update(class_kwargs(base_name))
+ initializer = next(
+ (node for node in class_node.body if isinstance(node, ast.FunctionDef) and node.name == "__init__"),
+ None,
+ )
+ if initializer is not None:
+ names.update(
+ argument.arg
+ for argument in (
+ *initializer.args.posonlyargs,
+ *initializer.args.args,
+ *initializer.args.kwonlyargs,
+ )
+ if argument.arg != "self"
+ )
+ if initializer.args.kwarg is not None:
+ names.update(literal_kwarg_accesses(initializer, name=initializer.args.kwarg.arg))
+ names.update(literal_kwarg_accesses(module_tree, attribute="_kwargs"))
+ resolved = frozenset(names)
+ resolved_classes[class_name] = resolved
+ return resolved
+
+ factory_tree = source_trees[environments_root / "factory.py"]
+ registry = next(
+ node.value
+ for node in factory_tree.body
+ if isinstance(node, ast.AnnAssign)
+ and isinstance(node.target, ast.Name)
+ and node.target.id == "_ENVIRONMENT_REGISTRY"
+ )
+ assert isinstance(registry, ast.Dict)
+
+ from harbor.models.environment_type import EnvironmentType
+
+ contract: dict[str, frozenset[str]] = {}
+ for key, value in zip(registry.keys, registry.values, strict=True):
+ assert isinstance(key, ast.Attribute)
+ assert isinstance(value, ast.Call)
+ class_name_node = value.args[1]
+ assert isinstance(class_name_node, ast.Constant) and isinstance(class_name_node.value, str)
+ contract[EnvironmentType[key.attr].value] = class_kwargs(class_name_node.value)
+ return contract
+
+
+def test_harbor_022_environment_kwarg_contract_matches_installed_source() -> None:
+ from skillevaluator import tier3_environments
+
+ contract = getattr(tier3_environments, "HARBOR_V022_ENVIRONMENT_KWARGS", None)
+
+ assert contract is not None
+ assert contract == _harbor_022_environment_kwargs_from_installed_source()
+
+
+def test_public_docs_match_harbor_environment_and_kwarg_contract() -> None:
+ agents = (REPO_ROOT / "docs" / "agents-and-sandboxes.mdx").read_text(encoding="utf-8")
+ cli_reference = (REPO_ROOT / "docs" / "cli-reference.mdx").read_text(encoding="utf-8")
+ configuration = (REPO_ROOT / "docs" / "configuration.mdx").read_text(encoding="utf-8")
+ tier3 = (REPO_ROOT / "docs" / "tier3-live-evaluation.mdx").read_text(encoding="utf-8")
+ eval_config = (REPO_ROOT / "docs" / "eval-datasets.mdx").read_text(encoding="utf-8")
+ public_docs = f"{agents}\n{cli_reference}\n{configuration}\n{tier3}\n{eval_config}"
+ normalized_docs = " ".join(public_docs.split())
+
+ assert "27 environment modes" in public_docs
+ assert "26 Harbor-native backends" in public_docs
+ assert "All 16 values" not in public_docs
+ assert "same 16 values" not in public_docs
+ assert "14 additional Harbor-native backends" not in public_docs
+ assert all(f"`{mode}`" in agents for mode in HARBOR_ENVIRONMENTS)
+ for mode, extra in HARBOR_ENVIRONMENT_EXTRAS.items():
+ if extra is not None:
+ assert f"| `{mode}` | Harbor-native | `harbor[{extra}]==0.22.0`" in agents
+ assert "`harbor[cloud]==0.22.0`" not in public_docs
+ assert "OpenSandbox requires either a non-empty `domain`" in normalized_docs
+ assert "does not contact AWS" in normalized_docs
+ assert "does not contact the GKE cluster" in normalized_docs
+ assert "no harbor python extra" in agents.lower()
+ assert "`--environment-kwarg`, `--ek`" in cli_reference
+ assert "repeatable" in cli_reference
+ assert "harbor.environment_kwargs" not in eval_config
+ assert all(name in public_docs for name in ("cluster_name", "registry_location", "ami_id", "instance_id"))
+ assert "never pass secrets" in public_docs
+ assert "only for Harbor-native non-Docker backends" in normalized_docs
+ assert "`docker` and `local` reject any environment kwargs" in normalized_docs
+ assert "operator-only" in normalized_docs
+ assert "skill-owned `evals/config.yml` cannot set" in normalized_docs
+ assert "Harbor runtime policy" in normalized_docs
+ assert all(
+ name in public_docs
+ for name in ("override_cpus", "mounts", "network_policy", "extra_docker_compose", "keep_containers")
+ )
+ kwarg_examples = [
+ block
+ for document in (agents, cli_reference, configuration, tier3, eval_config)
+ for block in re.findall(r"```(?:bash|shell|yaml)[^\n]*\n.*?```", document, flags=re.DOTALL)
+ if "--environment-kwarg " in block or "--ek " in block
+ ]
+ assert kwarg_examples
+ assert all(re.search(r"--env-mode (?:ec2|gke|ack)\b", block) for block in kwarg_examples)
+
+
def test_public_extras_exclude_internal_runtime_dependencies() -> None:
project = _project()
extras = project["project"]["optional-dependencies"]
@@ -195,6 +416,31 @@ def test_idna_is_a_bounded_direct_dependency_with_a_license_notice() -> None:
assert "IDNA (BSD-3-Clause)" in notices
+def test_tier3_direct_dependencies_have_complete_license_notices() -> None:
+ tier3 = [Requirement(raw) for raw in _project()["project"]["optional-dependencies"]["tier3"]]
+ direct_packages = {
+ canonicalize_name(requirement.name)
+ for requirement in tier3
+ if canonicalize_name(requirement.name) != "skillevaluator"
+ }
+ expected_notices = {
+ "harbor": "Harbor (Apache-2.0)",
+ "mcp": "MCP (MIT)",
+ "pyjwt": "PyJWT (MIT)",
+ }
+ notices = (REPO_ROOT / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8")
+
+ assert direct_packages == expected_notices.keys()
+ assert all(notice in notices for notice in expected_notices.values())
+
+
+def test_public_docker_docs_explain_project_dotenv_rejection() -> None:
+ agents_and_sandboxes = (REPO_ROOT / "docs" / "agents-and-sandboxes.mdx").read_text(encoding="utf-8")
+
+ assert "Docker Compose project `.env` files are rejected before startup" in agents_and_sandboxes
+ assert "`harbor.runtime_env`" in agents_and_sandboxes
+
+
def test_release_lock_avoids_accidental_prereleases_and_known_fixed_versions() -> None:
project = _project()
lock = _lock()
@@ -462,6 +708,15 @@ def test_ci_scans_source_and_built_distributions_for_oss_boundary_violations() -
assert workflow.index("uv build --python 3.13 --no-sources") < workflow.index(artifact_scan)
+def test_ci_installs_the_built_tier3_wheel_on_python_312_and_313() -> None:
+ workflow = (REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8")
+ package_job = workflow.split(" package:\n", 1)[1].split("\n tier3-macos:\n", 1)[0]
+
+ assert "for python_version in 3.12 3.13; do" in package_job
+ assert 'uv venv --python "$python_version" "$venv"' in package_job
+ assert 'uv pip install --python "$venv/bin/python" "${wheel}[tier3]"' in package_job
+
+
def test_retired_private_upload_artifact_is_not_part_of_public_gitignore() -> None:
gitignore = (REPO_ROOT / ".gitignore").read_text(encoding="utf-8")
retired_artifact = "." + "harbor" + "-viewer-upload/" # oss-boundary-anchor: gitignore-retired-upload-artifact
@@ -580,6 +835,27 @@ def test_tier3_docs_explain_cost_controls_and_local_mode_tradeoffs() -> None:
assert "weaker isolation than Docker" in tier3
+def test_tier3_docs_name_the_supported_harbor_backend_version() -> None:
+ tier3 = (REPO_ROOT / "docs" / "tier3-live-evaluation.mdx").read_text(encoding="utf-8")
+ normalized = " ".join(tier3.split())
+
+ assert "The `tier3` extra installs Harbor 0.22.0" in normalized
+ assert "The exact Harbor pin is deliberate" in normalized
+ assert "stable Harbor 0.22.0 rather than unreleased `main`" in normalized
+ assert "litellm>=1.92.0,<1.94.0.dev0" in normalized
+ assert "static Tier 1 installs do not pull it in" in normalized
+ assert "0.13.2" not in normalized
+
+
+def test_tier3_docs_describe_the_current_nvidia_build_docker_handoff() -> None:
+ sandboxes = (REPO_ROOT / "docs" / "agents-and-sandboxes.mdx").read_text(encoding="utf-8")
+ normalized = " ".join(sandboxes.split())
+
+ assert "host-only key file" not in normalized
+ assert "trusted parent process over stdin" in normalized
+ assert "short-lived, container-only stdin handoff" in normalized
+
+
def test_launch_docs_address_scanner_and_naming_ambiguities() -> None:
quickstart = (REPO_ROOT / "docs" / "quickstart.mdx").read_text(encoding="utf-8")
ci = (REPO_ROOT / "docs" / "ci-integration.mdx").read_text(encoding="utf-8")
diff --git a/tests/test_safety_boundaries.py b/tests/test_safety_boundaries.py
index 008bd10e..0ca872a6 100644
--- a/tests/test_safety_boundaries.py
+++ b/tests/test_safety_boundaries.py
@@ -5,10 +5,12 @@
from __future__ import annotations
+import json
+
import pytest
from skillevaluator.utils.process_environment import child_process_env
-from skillevaluator.utils.redaction import redact_sensitive_data, redact_sensitive_text
+from skillevaluator.utils.redaction import contains_credential_value, redact_sensitive_data, redact_sensitive_text
def _pem_block(
@@ -37,6 +39,61 @@ def test_redact_sensitive_text_masks_common_credentials() -> None:
assert redacted.count("") >= 4
+@pytest.mark.parametrize(
+ "value",
+ [
+ "case-sk-abcdefghijk",
+ "case_nvapi-abcdefghijk",
+ "case_ghp_" + ("a" * 36),
+ "case_gho_" + ("a" * 36),
+ "case_ghu_" + ("a" * 36),
+ "case_github_pat_" + ("a" * 30),
+ "case_ghr_" + ("a" * 36),
+ "caseghp_" + ("a" * 36),
+ "casegho_" + ("a" * 36) + "suffix",
+ "case_ghs_123456789_" + ("a" * 32) + "." + ("b" * 32) + "." + ("c" * 32),
+ "case_" + "".join(("xoxb-", "1234567890-abcdefghijklmnopqrstuvwx")), # noqa: FLY002
+ "case_" + "AIza" + ("A" * 35),
+ "case_glpat-" + ("a" * 20),
+ ],
+)
+def test_contains_credential_value_detects_embedded_identity_tokens(value: str) -> None:
+ assert contains_credential_value(value) is True
+
+
+def test_contains_credential_value_preserves_case_sensitive_noncanonical_prefix() -> None:
+ assert contains_credential_value("case-SK-ABCDEFGHIJK") is False
+
+
+def test_contains_credential_value_does_not_match_sk_inside_an_ordinary_word() -> None:
+ assert contains_credential_value("task-legacy-model") is False
+
+
+@pytest.mark.parametrize("prefix", ["ghp", "gho", "ghu", "ghs", "ghr"])
+def test_contains_credential_value_does_not_match_short_github_shaped_identifiers(prefix: str) -> None:
+ assert contains_credential_value(f"case_{prefix}_{'a' * 35}") is False
+
+
+@pytest.mark.parametrize(
+ "source",
+ (
+ "proxy https://alice:correct-horse-battery@example.test/path failed",
+ "proxy https://alice:correct@horse@example.test/path failed",
+ "proxy socks5://bearer-token@example.test failed",
+ "proxy http://user%3Apassword@example.test failed",
+ ),
+)
+def test_redact_sensitive_text_masks_uri_userinfo(source: str) -> None:
+ redacted = redact_sensitive_text(source)
+
+ assert "alice" not in redacted
+ assert "correct-horse-battery" not in redacted
+ assert "correct@horse" not in redacted
+ assert "bearer-token" not in redacted
+ assert "user%3Apassword" not in redacted
+ assert "@example.test" in redacted
+
+
def test_redact_sensitive_text_masks_unlabelled_secret_shapes() -> None:
aws_access_key = "".join(("AKIA", "IOSFODNN7", "EXAMPLE")) # noqa: FLY002 - scanner-safe fixture
jwt = ".".join(
@@ -169,6 +226,9 @@ def test_redact_sensitive_data_masks_secret_keys_and_nested_text() -> None:
"message": "Authorization: Bearer nested-secret-value",
},
"token_count": 42,
+ "last_token_usage": {"prompt_tokens": 10, "completion_tokens": 2},
+ "tokens": ["opaque-secret"],
+ "passwords": ["another-secret"],
}
assert redact_sensitive_data(source) == {
@@ -177,7 +237,27 @@ def test_redact_sensitive_data_masks_secret_keys_and_nested_text() -> None:
"message": "Authorization:",
},
"token_count": 42,
+ "last_token_usage": {"prompt_tokens": 10, "completion_tokens": 2},
+ "tokens": "",
+ "passwords": "",
+ }
+
+
+def test_redact_sensitive_data_drops_credential_values_used_as_mapping_keys() -> None:
+ credential = "sk-abcdefghijk"
+
+ redacted = redact_sensitive_data(
+ {
+ "custom_metrics": {credential: 0.5, "quality": 0.7},
+ "api_key": "provider-secret",
+ }
+ )
+
+ assert redacted == {
+ "custom_metrics": {"quality": 0.7},
+ "api_key": "",
}
+ assert credential not in json.dumps(redacted)
def test_child_process_env_strips_observability_configuration_without_injecting_flags() -> None:
diff --git a/tests/test_secure_docker_environment.py b/tests/test_secure_docker_environment.py
index 42f5d2bd..86af5345 100644
--- a/tests/test_secure_docker_environment.py
+++ b/tests/test_secure_docker_environment.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import asyncio
+import contextvars
import hashlib
import importlib
import importlib.util
@@ -20,6 +21,11 @@
import pytest
+def _initialize_harbor_context(environment: object) -> None:
+ environment._output_callbacks = contextvars.ContextVar("test_docker_output_callbacks", default=())
+ environment._exec_env_overlays = contextvars.ContextVar("test_docker_exec_env_overlays", default=())
+
+
def test_secure_docker_exec_streams_environment_without_host_file_or_argv_values(
monkeypatch,
tmp_path: Path,
@@ -28,6 +34,7 @@ def test_secure_docker_exec_streams_environment_without_host_file_or_argv_values
assert importlib.util.find_spec(module_name) is not None, "secure Docker environment module is missing"
module = importlib.import_module(module_name)
environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
+ _initialize_harbor_context(environment)
environment._persistent_env = {"PERSISTENT_TOKEN": "persistent-secret-value"}
environment.default_user = "1000"
environment.task_env_config = SimpleNamespace(workdir="/workspace")
@@ -45,16 +52,18 @@ async def fake_run(
command: list[str],
check: bool = True,
timeout_sec: int | None = None,
+ stdin_data: bytes | None = None,
+ on_output: object | None = None,
*,
- stdin_bytes: bytes | None = None,
- redact_values: set[str] | None = None,
+ additional_secret_values: set[str] | None = None,
+ exact_secret_values: set[str] | None = None,
stop_main_on_interrupt: bool = False,
):
- del check, timeout_sec
+ del check, timeout_sec, on_output, exact_secret_values
assert stop_main_on_interrupt is ("if ! ." in " ".join(command))
- if stdin_bytes is not None:
- handoff_payloads.append(stdin_bytes.decode("utf-8"))
- assert redact_values is not None
+ if stdin_data is not None:
+ handoff_payloads.append(stdin_data.decode("utf-8"))
+ assert additional_secret_values is not None
docker_commands.append(command)
return SimpleNamespace(stdout="ok", stderr=None, return_code=0)
@@ -88,6 +97,7 @@ def test_secure_docker_exec_redacts_persistent_and_per_call_credentials_from_all
) -> None:
module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
+ _initialize_harbor_context(environment)
persistent_secret = "persistent-credential-for-redaction"
per_call_secret = "per-call-credential-for-redaction"
environment._persistent_env = {"PERSISTENT_TOKEN": persistent_secret}
@@ -98,28 +108,56 @@ def test_secure_docker_exec_redacts_persistent_and_per_call_credentials_from_all
environment.environment_name = "secure-redaction-test"
environment.environment_dir = tmp_path
environment._resources_compose_path = None
+ environment._env_compose_path = None
environment._mounts_compose_path = None
environment._use_prebuilt = True
environment._is_windows_container = False
environment.extra_docker_compose_paths = []
environment._network_policy = SimpleNamespace(network_mode="public")
+ environment._enable_egress_control = False
+ environment._egress_control_services_compose_path = None
environment._compose_env_vars = lambda **_kwargs: {"PATH": "/usr/bin"}
+ environment._compose_infra_env_vars = dict
+ environment._windows_container_name = None
async def fake_upload_file(_source_path: Path | str, _target_path: str) -> None:
return None
+ class FakeInput:
+ def write(self, _data: bytes) -> None:
+ return None
+
+ async def drain(self) -> None:
+ return None
+
+ def close(self) -> None:
+ return None
+
+ async def wait_closed(self) -> None:
+ return None
+
+ class FakeOutput:
+ def __init__(self, payload: bytes) -> None:
+ self._payload = payload
+
+ async def read(self, _size: int) -> bytes:
+ payload, self._payload = self._payload, b""
+ return payload
+
class FakeProcess:
returncode = 0
def __init__(self, *, expose_secrets: bool) -> None:
- self._expose_secrets = expose_secrets
+ payload = b""
+ if expose_secrets:
+ output = f"stdout {persistent_secret} {per_call_secret}".encode()
+ error = f"stderr {per_call_secret} {persistent_secret}".encode()
+ payload = output + error
+ self.stdin = FakeInput()
+ self.stdout = FakeOutput(payload)
- async def communicate(self, _input: bytes | None = None) -> tuple[bytes, bytes]:
- if not self._expose_secrets:
- return b"", b""
- output = f"stdout {persistent_secret} {per_call_secret}".encode()
- error = f"stderr {per_call_secret} {persistent_secret}".encode()
- return output, error
+ async def wait(self) -> int:
+ return self.returncode
async def create_subprocess(*args: object, **_kwargs: object) -> FakeProcess:
rendered = " ".join(str(arg) for arg in args)
@@ -149,10 +187,11 @@ def test_nvidia_build_docker_command_uses_secure_environment_and_bridge() -> Non
agent_import_path=agent_import_path,
)
- assert command[command.index("--agent-import-path") + 1] == agent_import_path
- assert command[command.index("--environment-import-path") + 1] == runner.SECURE_DOCKER_ENV_IMPORT_PATH
+ assert "--agent-import-path" not in command
+ assert "--environment-import-path" not in command
assert "-a" not in command
- assert "--env" not in command
+ assert command[command.index("--agent") + 1] == agent_import_path
+ assert command[command.index("--env") + 1] == runner.SECURE_DOCKER_ENV_IMPORT_PATH
def test_secure_docker_exec_cleans_remote_handoff_when_streaming_reports_failure(
@@ -162,6 +201,7 @@ def test_secure_docker_exec_cleans_remote_handoff_when_streaming_reports_failure
del tmp_path
module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
+ _initialize_harbor_context(environment)
environment._persistent_env = {}
environment.default_user = "1000"
environment.task_env_config = SimpleNamespace(workdir="/workspace")
@@ -172,11 +212,11 @@ async def stream_then_fail(
_command: list[str],
_check: bool = True,
_timeout_sec: int | None = None,
- *,
- stdin_bytes: bytes | None = None,
+ stdin_data: bytes | None = None,
+ _on_output: object | None = None,
**_kwargs,
) -> SimpleNamespace:
- if stdin_bytes is not None:
+ if stdin_data is not None:
raise ConnectionError("docker exec disconnected after creating the file")
return SimpleNamespace(stdout="", stderr=None, return_code=0)
@@ -197,6 +237,7 @@ def test_secure_docker_exec_repeated_cancellation_does_not_interrupt_handoff_cle
) -> None:
module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
+ _initialize_harbor_context(environment)
environment._persistent_env = {}
environment.default_user = "1000"
environment.task_env_config = SimpleNamespace(workdir="/workspace")
@@ -212,11 +253,11 @@ async def blocked_stream(
_command: list[str],
_check: bool = True,
_timeout_sec: int | None = None,
- *,
- stdin_bytes: bytes | None = None,
+ stdin_data: bytes | None = None,
+ _on_output: object | None = None,
**_kwargs,
) -> SimpleNamespace:
- if stdin_bytes is not None:
+ if stdin_data is not None:
upload_started.set()
await asyncio.Event().wait()
return SimpleNamespace(stdout="", stderr=None, return_code=0)
@@ -246,6 +287,7 @@ async def remove_handoff(_remote_path: str) -> None:
def test_secure_docker_exec_fails_closed_when_final_secret_cleanup_fails(monkeypatch) -> None:
module = importlib.import_module("skillevaluator.tier3.harbor.secure_docker_environment")
environment = object.__new__(module.SkillEvaluatorSecureDockerEnvironment)
+ _initialize_harbor_context(environment)
environment._persistent_env = {}
environment.default_user = "1000"
environment.task_env_config = SimpleNamespace(workdir="/workspace")
@@ -258,12 +300,21 @@ async def fake_run(
command: list[str],
check: bool = True,
timeout_sec: int | None = None,
+ stdin_data: bytes | None = None,
+ on_output: object | None = None,
*,
- stdin_bytes: bytes | None = None,
- redact_values: set[str] | None = None,
+ additional_secret_values: set[str] | None = None,
+ exact_secret_values: set[str] | None = None,
stop_main_on_interrupt: bool = False,
):
- del check, timeout_sec, stdin_bytes, redact_values
+ del (
+ check,
+ timeout_sec,
+ stdin_data,
+ on_output,
+ additional_secret_values,
+ exact_secret_values,
+ )
assert stop_main_on_interrupt is ("if ! ." in " ".join(command))
return SimpleNamespace(stdout="ok", stderr=None, return_code=0)
@@ -309,10 +360,10 @@ def fake_run(command, **kwargs):
assert environment["NVIDIA_API_KEY"] == runner._NVIDIA_BUILD_STDIN_SENTINEL
assert environment[runner._NVIDIA_BUILD_KEY_STDIN_ENV] == "1"
assert runner._NVIDIA_BUILD_KEY_FILE_ENV not in environment
- assert kwargs["input"] == secret
- return SimpleNamespace(returncode=0, stdout="", stderr="")
+ assert kwargs["stdin_text"] == secret
+ return runner._BoundedHarborProcessResult(returncode=0, output_tail="", output_exceeded=False)
- monkeypatch.setattr(runner.subprocess, "run", fake_run)
+ monkeypatch.setattr(runner, "_run_bounded_harbor_process", fake_run)
monkeypatch.setattr(runner, "_validate_harbor_job_result", lambda *_args, **_kwargs: (True, ""))
ok, detail = runner._run_harbor(
@@ -356,11 +407,15 @@ def test_harbor_subprocess_redacts_stdin_key_from_early_failure(monkeypatch, tmp
secret = "nvidia-real-secret-value-for-test"
def fake_run(command, **kwargs):
- assert kwargs["input"] == secret
+ assert kwargs["stdin_text"] == secret
assert secret not in kwargs["env"].values()
- return SimpleNamespace(returncode=1, stdout="", stderr=f"provider rejected {secret}")
+ return runner._BoundedHarborProcessResult(
+ returncode=1,
+ output_tail=f"provider rejected {secret}",
+ output_exceeded=False,
+ )
- monkeypatch.setattr(runner.subprocess, "run", fake_run)
+ monkeypatch.setattr(runner, "_run_bounded_harbor_process", fake_run)
ok, detail = runner._run_harbor(
dataset=tmp_path / "dataset",
diff --git a/tests/test_tier3_config_key_parity.py b/tests/test_tier3_config_key_parity.py
index b243fcfc..934ff7b0 100644
--- a/tests/test_tier3_config_key_parity.py
+++ b/tests/test_tier3_config_key_parity.py
@@ -90,11 +90,12 @@ def collect(**kwargs: Any) -> dict[str, Any]:
),
)
agents = engine_kwargs.pop("agents", ["opencode"])
+ env_mode = engine_kwargs.pop("env_mode", "docker")
result = runner.run_harbor_eval(
skill,
agents,
output_dir=tmp_path / "results",
- env_mode="docker",
+ env_mode=env_mode,
agent_runtime_preflight=engine_kwargs.pop("agent_runtime_preflight", False),
**engine_kwargs,
)
@@ -122,6 +123,41 @@ def test_user_config_parses_and_routes_end_to_end(monkeypatch: pytest.MonkeyPatc
assert captured["collect"]["expected_trials"] == 1
+def test_environment_kwargs_cli_routes_to_every_harbor_run(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ config = """\
+schema_version: 1
+harbor:
+ task_source: evals_json
+"""
+
+ result, captured = _run_engine(
+ monkeypatch,
+ tmp_path,
+ config,
+ env_mode="ec2",
+ environment_kwargs={"region": "us-west-2", "launch_mode": "attach", "instance_id": "i-123"},
+ )
+
+ expected = {
+ "region": "us-west-2",
+ "launch_mode": "attach",
+ "instance_id": "i-123",
+ }
+ assert "error" not in result
+ assert captured["pair"]["environment_kwargs"] == expected
+ assert result["run_config"]["harbor"]["environment_kwargs"] == {
+ "keys": ["instance_id", "launch_mode", "region"],
+ "sources": {
+ "instance_id": "CLI",
+ "launch_mode": "CLI",
+ "region": "CLI",
+ },
+ }
+
+
def test_stop_on_pass_config_key_routes_sequential_attempt_policy(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -235,6 +271,33 @@ def test_agent_runtime_preflight_defaults_to_enabled(
assert calls == ["opencode"]
+def test_environment_kwargs_route_to_runtime_preflight(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ observed: dict[str, Any] = {}
+
+ def observe(**kwargs: Any) -> SimpleNamespace:
+ observed.update(kwargs)
+ return SimpleNamespace(ok=True, detail="")
+
+ monkeypatch.setattr(runtime_preflight, "run_agent_runtime_preflight", observe)
+ result, _captured = _run_engine(
+ monkeypatch,
+ tmp_path,
+ USER_CONFIG,
+ env_mode="e2b",
+ agent_runtime_preflight=True,
+ environment_kwargs={"region": "us-west-2", "labels": {"owner": "eval"}},
+ )
+
+ assert "error" not in result
+ assert observed["environment_kwargs"] == {
+ "region": "us-west-2",
+ "labels": {"owner": "eval"},
+ }
+
+
def test_agent_runtime_preflight_cli_value_overrides_config(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
diff --git a/tests/test_tier3_progress.py b/tests/test_tier3_progress.py
index a2734761..d8dbfe6d 100644
--- a/tests/test_tier3_progress.py
+++ b/tests/test_tier3_progress.py
@@ -10,7 +10,6 @@
import io
import json
import os
-import subprocess
import tempfile
import threading
from concurrent.futures import ThreadPoolExecutor
@@ -485,6 +484,77 @@ def test_reporters_redact_exact_values_and_secret_shaped_details() -> None:
assert "" in rendered
+@pytest.mark.parametrize(
+ "proxy_uri",
+ [
+ "http://proxy-user:proxy-password@proxy.example:8080",
+ "https://proxy-user:password@fragment@proxy.example:8443",
+ "https://bearer-token@proxy.example",
+ "socks5://user%3Apassword@proxy.example:1080",
+ ],
+)
+def test_progress_redacts_proxy_uri_userinfo_even_before_secret_values_are_registered(proxy_uri: str) -> None:
+ progress = _progress_module()
+
+ extracted = progress.secret_values_from_environment({"HTTP_PROXY": proxy_uri})
+ rendered = progress.redact_progress_detail(f"Cannot connect to proxy {proxy_uri}", secret_values=set())
+
+ assert proxy_uri in extracted
+ assert "proxy-user" not in rendered
+ assert "proxy-password" not in rendered
+ assert "password@fragment" not in rendered
+ assert "bearer-token" not in rendered
+ assert "user%3Apassword" not in rendered
+ assert "@proxy.example" in rendered
+
+
+@pytest.mark.parametrize(
+ ("proxy_uri", "diagnostic", "secrets"),
+ [
+ (
+ "http://proxy-user:proxy-password@proxy.example:8080",
+ "proxy auth failed for proxy-user with proxy-password",
+ ("proxy-user", "proxy-password"),
+ ),
+ (
+ "http://proxy%2Duser:proxy%2Dpassword@proxy.example:8080",
+ "proxy auth failed for proxy-user with proxy-password",
+ ("proxy-user", "proxy-password"),
+ ),
+ (
+ "proxy-user:proxy-password@proxy.example:8080",
+ "proxy auth failed for proxy-user with proxy-password",
+ ("proxy-user", "proxy-password"),
+ ),
+ ],
+)
+def test_progress_registers_proxy_userinfo_components(
+ proxy_uri: str,
+ diagnostic: str,
+ secrets: tuple[str, ...],
+) -> None:
+ progress = _progress_module()
+
+ extracted = progress.secret_values_from_environment({"HTTPS_PROXY": proxy_uri})
+ rendered = progress.redact_progress_detail(diagnostic, secret_values=extracted)
+
+ assert all(secret in extracted for secret in secrets)
+ assert all(secret not in rendered for secret in secrets)
+
+
+@pytest.mark.parametrize("short_secret", ["x", "pw", "usr"])
+def test_progress_redacts_short_exact_credential_fragments_as_standalone_tokens(short_secret: str) -> None:
+ progress = _progress_module()
+ rendered = progress.redact_progress_detail(
+ f"proxy rejected credential {short_secret}; password remains a normal word",
+ secret_values={short_secret},
+ )
+
+ assert f" {short_secret};" not in rendered
+ assert "" in rendered
+ assert "password remains" in rendered
+
+
def test_progress_detail_strips_osc_title_and_hyperlink_payloads() -> None:
progress = _progress_module()
@@ -498,6 +568,15 @@ def test_progress_detail_strips_osc_title_and_hyperlink_payloads() -> None:
assert rendered == "safe text click done"
+def test_progress_redaction_scales_linearly_for_large_non_uri_text() -> None:
+ progress = _progress_module()
+ payload = "x" * (1024 * 1024)
+
+ rendered = progress.redact_progress_detail(payload)
+
+ assert rendered == payload
+
+
def test_plain_reporter_redacts_secrets_from_plan_values() -> None:
progress = _progress_module()
output = io.StringIO()
@@ -696,6 +775,44 @@ def write_html(_skill_path, run_dir, **_kwargs):
return runner, skill
+def test_run_result_bounds_multibyte_launch_errors_for_report_loader(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ github_token = "ghp_" + ("A" * 36)
+ launch_errors = [f"launch {index}: {github_token}\x1b[2J" + ("😀" * 2_048) for index in range(256)]
+ runner, skill = _stub_runner(
+ monkeypatch,
+ tmp_path,
+ run_agent=lambda **_kwargs: launch_errors,
+ )
+
+ result = runner.run_harbor_eval(
+ skill,
+ ["codex"],
+ output_dir=tmp_path / "results",
+ agent_runtime_preflight=False,
+ )
+
+ result_path = Path(result["result_path"])
+ serialized = result_path.read_bytes()
+ persisted = json.loads(serialized)
+ assert len(serialized) <= 2 * 1024 * 1024
+ assert persisted["execution_status"] == "failed"
+ assert persisted["execution_error_details_total"] == len(launch_errors)
+ assert persisted["execution_error_details_shown"] == len(persisted["execution_errors"])
+ assert persisted["execution_error_details_truncated"] is True
+ assert persisted["error"] == persisted["execution_errors"][:1]
+ assert github_token not in serialized.decode("utf-8")
+ assert "\\u001b" not in serialized.decode("utf-8")
+
+ from skillevaluator.tier3.harbor import report_data
+
+ diagnostics: list[dict[str, Any]] = []
+ assert report_data._load_bounded_json(result_path, diagnostics, artifact="result") == persisted
+ assert diagnostics == []
+
+
def _configure_native_task_source(
monkeypatch: pytest.MonkeyPatch,
runner,
@@ -735,6 +852,76 @@ def emit_native(_skill, output: Path, **_kwargs):
monkeypatch.setattr(runner, "stage_native_harbor_tasks", emit_native)
+def test_ack_eval_preflight_uses_the_exact_prospective_bedrock_child_environment(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.provider_config import ProviderConfig
+ from skillevaluator.tier3.harbor import runner as runner_module
+
+ real_harbor_subprocess_environment = runner_module._harbor_subprocess_environment
+ real_provider_environment = runner_module._provider_environment
+ captured: dict[str, Any] = {}
+ launched_env: dict[str, str] = {}
+
+ def capture_preflight(**kwargs: Any) -> list[str]:
+ captured.update(kwargs)
+ return []
+
+ def capture_launch(**kwargs: Any) -> list[str]:
+ launched_env.update(kwargs["run_env"])
+ return []
+
+ runner, skill = _stub_runner(
+ monkeypatch,
+ tmp_path,
+ environment_check=capture_preflight,
+ run_agent=capture_launch,
+ provider_name="bedrock",
+ provider_model="us.anthropic.claude-test",
+ )
+ provider = ProviderConfig(
+ provider="bedrock",
+ model="us.anthropic.claude-test",
+ api_key=None,
+ base_url=None,
+ litellm_model="bedrock/us.anthropic.claude-test",
+ region="us-west-2",
+ )
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "eks-exec-auth-key")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "eks-exec-auth-secret")
+ monkeypatch.setenv("KUBECONFIG", "/config/eks")
+ monkeypatch.setenv("SKILL_EVAL_JUDGE_MODEL", "judge-model")
+ monkeypatch.setenv("ALIBABA_CLOUD_ACCESS_KEY_ID", "ambient-parent-only")
+ monkeypatch.setattr(runner, "resolve_llm_provider", lambda: provider)
+ monkeypatch.setattr(runner, "_provider_environment", real_provider_environment)
+ monkeypatch.setattr(runner, "_harbor_subprocess_environment", real_harbor_subprocess_environment)
+ monkeypatch.setattr(runner, "_resolve_runtime_env", lambda _templates: ({"SAFE_RUNTIME_FLAG": "enabled"}, []))
+
+ result = runner.run_harbor_eval(
+ skill,
+ ["claude-code"],
+ env_mode="ack",
+ environment_kwargs={"namespace": "skill-evals"},
+ output_dir=tmp_path / "results",
+ agent_runtime_preflight=False,
+ )
+
+ assert result["execution_status"] == "succeeded"
+ child_env = captured["subprocess_env"]
+ assert isinstance(child_env, dict)
+ assert child_env == launched_env
+ assert child_env["KUBECONFIG"] == "/config/eks"
+ assert child_env["AWS_ACCESS_KEY_ID"] == "eks-exec-auth-key"
+ assert child_env["AWS_SECRET_ACCESS_KEY"] == "eks-exec-auth-secret"
+ assert child_env["AWS_REGION"] == "us-west-2"
+ assert child_env["CLAUDE_CODE_USE_BEDROCK"] == "1"
+ assert child_env["SAFE_RUNTIME_FLAG"] == "enabled"
+ assert child_env["LLM_JUDGE_MODEL"] == "judge-model"
+ assert child_env["SKILL_EVAL_JUDGE_MODEL"] == "judge-model"
+ assert "ALIBABA_CLOUD_ACCESS_KEY_ID" not in child_env
+
+
@pytest.mark.parametrize("agent_runtime_preflight", [True, False])
def test_credential_validation_401_stops_before_image_and_task_preparation(
monkeypatch: pytest.MonkeyPatch,
@@ -2574,13 +2761,51 @@ def test_harbor_failure_detail_redacts_runtime_secrets_without_streaming(
secret = "plain-runtime-secret"
monkeypatch.setattr(runner, "build_harbor_run_command", lambda **_kwargs: ["harbor", "run"])
monkeypatch.setattr(
- runner.subprocess,
- "run",
- lambda *_args, **_kwargs: subprocess.CompletedProcess(
- ["harbor", "run"],
- 17,
- stdout=f"OPENAI_API_KEY={secret}\nraw agent output",
- stderr=f"token={secret}\nraw verifier output",
+ runner,
+ "_run_bounded_harbor_process",
+ lambda *_args, **_kwargs: runner._BoundedHarborProcessResult(
+ returncode=17,
+ output_tail=f"OPENAI_API_KEY={secret}\nraw agent output\ntoken={secret}\nraw verifier output",
+ output_exceeded=False,
+ ),
+ )
+
+ ok, detail = runner._run_harbor(
+ dataset=tmp_path / "dataset",
+ agent="codex",
+ job_name="demo-codex-with",
+ env_mode="docker",
+ model="gpt-5",
+ jobs_dir=tmp_path,
+ run_env={"CUSTOM_RUNTIME_VALUE": secret},
+ n_attempts=1,
+ n_concurrent=1,
+ timeout_multiplier=1.0,
+ override_cpus=None,
+ override_memory_mb=None,
+ override_storage_mb=None,
+ )
+
+ assert ok is False
+ assert secret not in detail
+ assert "" in detail
+
+
+def test_harbor_failure_detail_redacts_before_tail_truncation(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ from skillevaluator.tier3.harbor import runner
+
+ secret = "SYNTHETIC_SECRET_ABCDEF"
+ monkeypatch.setattr(runner, "build_harbor_run_command", lambda **_kwargs: ["harbor", "run"])
+ monkeypatch.setattr(
+ runner,
+ "_run_bounded_harbor_process",
+ lambda *_args, **_kwargs: runner._BoundedHarborProcessResult(
+ returncode=17,
+ output_tail="prefix|" + secret + "x" * 1988,
+ output_exceeded=False,
),
)
@@ -2602,6 +2827,7 @@ def test_harbor_failure_detail_redacts_runtime_secrets_without_streaming(
assert ok is False
assert secret not in detail
+ assert "CRET_ABCDEF" not in detail
assert "" in detail
diff --git a/tests/test_tier3_public_runtime.py b/tests/test_tier3_public_runtime.py
index 05e9fbab..3d7652f1 100644
--- a/tests/test_tier3_public_runtime.py
+++ b/tests/test_tier3_public_runtime.py
@@ -26,6 +26,9 @@
from skillevaluator.tier3.evals_config import EvalsConfigError, load_evals_config
from skillevaluator.tier3.harbor.adapter import _EVALUATOR_MANAGED_RUNTIME_ENV, _write_task_toml
from skillevaluator.tier3.harbor.runner import (
+ _check_prerequisites,
+ _environment_extra_install_hint,
+ _environment_kwarg_prerequisite_errors,
_model_for_agent,
_nvidia_build_agent_import_path,
_provider_environment,
@@ -33,6 +36,7 @@
build_harbor_run_command,
)
from skillevaluator.tier3.harbor.runtime_preflight import ModelProbeResult
+from skillevaluator.tier3_environments import HARBOR_NATIVE_ENV_MODES
def _load_verifier_template():
@@ -114,8 +118,491 @@ def test_native_environment_is_forwarded_to_harbor() -> None:
)
assert command[1] == "run"
- assert command[command.index("--env") + 1] == "e2b"
+ assert "--agent-import-path" not in command
assert "--environment-import-path" not in command
+ assert "-a" not in command
+ assert command.count("--agent") == 1
+ assert command[command.index("--agent") + 1] == "codex"
+ assert command.count("--env") == 1
+ assert command[command.index("--env") + 1] == "e2b"
+
+
+@pytest.mark.parametrize("timeout_multiplier", [float("nan"), float("inf"), float("-inf")])
+def test_harbor_command_rejects_nonfinite_timeout_multiplier(timeout_multiplier: float) -> None:
+ with pytest.raises(ValueError, match="timeout_multiplier must be a finite number greater than 0"):
+ build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="nonfinite-timeout",
+ env_mode="docker",
+ timeout_multiplier=timeout_multiplier,
+ )
+
+
+def test_harbor_command_rejects_overflowing_timeout_multiplier() -> None:
+ with pytest.raises(ValueError, match="timeout_multiplier must be a finite number greater than 0"):
+ build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="overflowing-timeout",
+ env_mode="docker",
+ timeout_multiplier=10**1000,
+ )
+
+
+def test_harbor_command_rejects_finite_multiplier_that_overflows_default_timeouts() -> None:
+ with pytest.raises(ValueError, match="must yield finite Harbor timeouts"):
+ build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="finite-overflowing-timeout",
+ env_mode="docker",
+ timeout_multiplier=1e308,
+ )
+
+
+def test_native_environment_kwargs_round_trip_through_real_harbor_parser() -> None:
+ from harbor.cli.utils import parse_kwargs
+
+ expected = {
+ "region": "us-west-2",
+ "security_group_ids": ["sg-123", "sg-456"],
+ "use_public_ip": False,
+ "root_volume_size_gb": 80,
+ }
+ command = build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="native-environment-kwargs",
+ env_mode="ec2",
+ environment_kwargs=expected,
+ )
+
+ encoded = [command[index + 1] for index, value in enumerate(command) if value == "--ek"]
+ assert parse_kwargs(encoded) == expected
+
+
+@pytest.mark.parametrize("env_mode", sorted(HARBOR_NATIVE_ENV_MODES - {"docker"}))
+def test_native_environment_kwargs_reject_unknown_harbor_022_names(env_mode: str) -> None:
+ with pytest.raises(ValueError, match=rf"Harbor 0\.22\.0 environment '{env_mode}'.*totally_ignored"):
+ build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="unknown-environment-kwarg",
+ env_mode=env_mode,
+ environment_kwargs={"totally_ignored": True},
+ )
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "name", "value"),
+ [
+ ("daytona", "connection_pool_maxsize", 32),
+ ("modal", "modal_vm_runtime", True),
+ ("novita", "dind_dockerd_start_cmd", "dockerd-entrypoint.sh dockerd"),
+ ],
+)
+def test_native_environment_hidden_harbor_022_kwargs_remain_usable(
+ env_mode: str,
+ name: str,
+ value: object,
+) -> None:
+ command = build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="hidden-environment-kwarg",
+ env_mode=env_mode,
+ environment_kwargs={name: value},
+ )
+
+ assert command[command.index("--ek") + 1].startswith(f"{name}=")
+
+
+def test_native_environment_kwargs_resolve_real_harbor_ec2_constructor(tmp_path: Path) -> None:
+ from harbor.cli.utils import parse_kwargs
+ from harbor.environments.factory import EnvironmentFactory
+ from harbor.models.environment_type import EnvironmentType
+ from harbor.models.task.config import EnvironmentConfig as TaskEnvironmentConfig
+ from harbor.models.trial.config import EnvironmentConfig as TrialEnvironmentConfig
+ from harbor.models.trial.paths import TrialPaths
+
+ expected = {
+ "region": "us-west-2",
+ "launch_mode": "attach",
+ "instance_id": "i-123",
+ }
+ command = build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="native-environment-constructor",
+ env_mode="ec2",
+ environment_kwargs=expected,
+ )
+ encoded = [command[index + 1] for index, value in enumerate(command) if value == "--ek"]
+ environment_dir = tmp_path / "environment"
+ environment_dir.mkdir()
+ (environment_dir / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
+ trial_dir = tmp_path / "trial"
+ trial_dir.mkdir()
+
+ environment = EnvironmentFactory.create_environment_from_config(
+ TrialEnvironmentConfig(type=EnvironmentType.EC2, kwargs=parse_kwargs(encoded)),
+ environment_dir=environment_dir,
+ environment_name="native-environment-constructor",
+ session_id="test-session",
+ trial_paths=TrialPaths(trial_dir),
+ task_env_config=TaskEnvironmentConfig(),
+ )
+
+ assert type(environment).__name__ == "EC2Environment"
+ assert environment.region == "us-west-2"
+ assert environment.launch_mode == "attach"
+ assert environment.instance_id == "i-123"
+
+
+def test_ack_operator_kwargs_allow_safe_registry_and_scheduling_references() -> None:
+ from harbor.cli.utils import parse_kwargs
+
+ expected = {
+ "namespace": "skill-evals",
+ "image_pull_secret": "registry-credentials",
+ "node_selector": {"pool": "sandbox"},
+ "tolerations": [{"key": "sandbox", "operator": "Exists"}],
+ }
+ command = build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="ack-operator-kwargs",
+ env_mode="ack",
+ environment_kwargs=expected,
+ )
+
+ encoded = [command[index + 1] for index, value in enumerate(command) if value == "--ek"]
+ assert parse_kwargs(encoded) == expected
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "environment_kwargs", "error"),
+ [
+ ("local", {"region": "us-west-2"}, "not supported for SkillEvaluator local mode"),
+ ("local", {"totally_ignored": True}, "not supported for SkillEvaluator local mode"),
+ ("docker", {"region": "us-west-2"}, "not supported for SkillEvaluator Docker mode"),
+ ("docker", {"totally_ignored": True}, "not supported for SkillEvaluator Docker mode"),
+ ("ec2", {"override_cpus": 999}, "reserved for Harbor runtime policy"),
+ ("ec2", {"extra_docker_compose": ["escape.yml"]}, "reserved for Harbor runtime policy"),
+ ("ec2", {"network_policy": {"network_mode": "public"}}, "reserved for Harbor runtime policy"),
+ ("ack", {"pod_overrides": {"spec": {"hostNetwork": True}}}, "reserved for Harbor runtime policy"),
+ ("ack", {"pod_privileged": True}, "reserved for Harbor runtime policy"),
+ ("ack", {"extra_volumes": [{"hostPath": {"path": "/"}}]}, "reserved for Harbor runtime policy"),
+ ],
+)
+def test_environment_kwargs_cannot_override_sandbox_or_runtime_policy(
+ env_mode: str,
+ environment_kwargs: dict[str, object],
+ error: str,
+) -> None:
+ with pytest.raises(ValueError, match=error):
+ build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="untrusted-environment-kwargs",
+ env_mode=env_mode,
+ environment_kwargs=environment_kwargs,
+ )
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "name", "value"),
+ [
+ ("ack", "build_job_namespace", "privileged-builds"),
+ ("ack", "buildkit_address", "tcp://buildkit.internal:1234"),
+ ("ack", "dind_image", "untrusted/dind:latest"),
+ ("ack", "memory_limit_multiplier", 0),
+ ("ack", "pod_annotations", {"inject-sidecar": "enabled"}),
+ ("ack", "pod_labels", {"network-policy": "bypass"}),
+ ("ack", "sandbox_env_vars", {"LD_PRELOAD": "/escape.so"}),
+ ("ack", "service_account", "cluster-admin"),
+ ("ack", "use_buildkit", True),
+ ("blaxel", "dind_extra_args", {"host": "tcp://0.0.0.0:2375"}),
+ ("cua-cloud", "claim_spec", {"serviceAccountName": "cluster-admin"}),
+ ("daytona", "network_block_all", False),
+ ("ec2", "iam_instance_profile", "administrator"),
+ ("ec2", "strict_host_key_checking", "no"),
+ ("gke", "memory_limit_multiplier", 0),
+ ("modal", "volumes", {"/workspace": "shared"}),
+ ("opensandbox", "volumes", [{"host_path": "/"}]),
+ ("openshift", "service_account_name", "cluster-admin"),
+ ("singularity", "singularity_no_mount", ""),
+ ("use-computer", "resources", {"cpu": 128, "memory": 1048576}),
+ ("vercel", "ports", [22, 2375]),
+ ],
+)
+def test_backend_aliases_cannot_bypass_sandbox_runtime_policy(
+ env_mode: str,
+ name: str,
+ value: object,
+) -> None:
+ with pytest.raises(ValueError, match=rf"reserved for Harbor runtime policy: {name}"):
+ build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="backend-policy-alias",
+ env_mode=env_mode,
+ environment_kwargs={name: value},
+ )
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "environment_kwargs"),
+ [
+ (
+ "ack",
+ {
+ "namespace": "skill-evals",
+ "use_sandbox_claim": True,
+ "sandbox_image": "registry.example/harbor-sandbox:v1",
+ # SandboxSet template metadata is an intentional operator
+ # integration surface, unlike legacy direct pod overrides.
+ "sandbox_labels": {"pool": "eval"},
+ "sandbox_annotations": {"owner": "operator"},
+ "skip_image_check": False,
+ },
+ ),
+ (
+ "ec2",
+ {
+ "region": "us-west-2",
+ "ami_id": "ami-123",
+ "instance_type": "m7i-flex.large",
+ "root_volume_size_gb": 80,
+ "bootstrap_docker": False,
+ },
+ ),
+ (
+ "gke",
+ {
+ "cluster_name": "cluster",
+ "region": "us-central1",
+ "namespace": "skill-evals",
+ "registry_location": "us-central1",
+ "registry_name": "skill-evals",
+ "cloud_build_machine_type": "E2_HIGHCPU_32",
+ "cloud_build_disk_size_gb": 500,
+ },
+ ),
+ (
+ "opensandbox",
+ {
+ "entrypoint": ["/bin/sh", "-lc", "sleep infinity"],
+ "extensions": {"provider.example/feature": "enabled"},
+ "sandbox_timeout_sec": 7200,
+ },
+ ),
+ ],
+)
+def test_backend_operator_functionality_outside_policy_boundary_remains_usable(
+ env_mode: str,
+ environment_kwargs: dict[str, object],
+) -> None:
+ command = build_harbor_run_command(
+ dataset_path="/tmp/dataset",
+ agent="codex",
+ job_name="allowed-backend-options",
+ env_mode=env_mode,
+ environment_kwargs=environment_kwargs,
+ )
+
+ assert command.count("--ek") == len(environment_kwargs)
+
+
+def test_skill_config_cannot_supply_environment_kwargs(tmp_path: Path) -> None:
+ evals = tmp_path / "evals"
+ evals.mkdir()
+ (evals / "config.yml").write_text(
+ "schema_version: 1\n"
+ "harbor:\n"
+ " environment_kwargs:\n"
+ " extra_docker_compose:\n"
+ " - /tmp/privileged-compose.yml\n",
+ encoding="utf-8",
+ )
+ with pytest.raises(EvalsConfigError, match=r"unknown harbor key.*environment_kwargs"):
+ load_evals_config(tmp_path)
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "environment_kwargs", "expected"),
+ [
+ ("ec2", {}, "region"),
+ ("ec2", {"region": "us-west-2"}, "ami_id"),
+ ("ec2", {"region": "us-west-2", "launch_mode": "attach"}, "instance_id"),
+ (
+ "gke",
+ {"cluster_name": "cluster", "region": "us-west1", "namespace": "evals"},
+ "registry_location, registry_name",
+ ),
+ ("ack", {}, "namespace"),
+ ],
+)
+def test_native_environment_required_kwargs_fail_before_mutation(
+ env_mode: str,
+ environment_kwargs: dict[str, object],
+ expected: str,
+) -> None:
+ assert expected in _environment_kwarg_prerequisite_errors(env_mode, environment_kwargs)[0]
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "environment_kwargs", "expected"),
+ [
+ (
+ "gke",
+ {
+ "cluster_name": 1,
+ "region": [],
+ "namespace": {},
+ "registry_location": False,
+ "registry_name": "valid",
+ },
+ "cluster_name, region, namespace, registry_location",
+ ),
+ ("ack", {"namespace": []}, "namespace"),
+ ("ec2", {"region": False, "ami_id": "ami-123"}, "region"),
+ ("ec2", {"region": "us-west-2", "ami_id": 123}, "ami_id"),
+ ("ec2", {"region": "us-west-2", "launch_mode": [], "instance_id": "i-123"}, "launch_mode"),
+ ("ec2", {"region": "us-west-2", "launch_mode": "attach", "instance_id": {}}, "instance_id"),
+ ],
+)
+def test_native_environment_required_kwargs_reject_non_string_values_without_crashing(
+ env_mode: str,
+ environment_kwargs: dict[str, object],
+ expected: str,
+) -> None:
+ errors = _environment_kwarg_prerequisite_errors(env_mode, environment_kwargs)
+
+ assert len(errors) == 1
+ assert expected in errors[0]
+
+
+def test_native_environment_required_kwargs_accept_valid_ec2_attach_configuration() -> None:
+ assert (
+ _environment_kwarg_prerequisite_errors(
+ "ec2",
+ {"region": "us-west-2", "launch_mode": "attach", "instance_id": "i-123"},
+ )
+ == []
+ )
+
+
+@pytest.mark.parametrize(
+ "ssh_key_path",
+ ["", "/definitely/missing/harbor-ssh-key", "~definitely-no-such-user-issue79/key"],
+)
+def test_ec2_environment_kwargs_reject_nonexistent_ssh_key_path(ssh_key_path: str) -> None:
+ errors = _environment_kwarg_prerequisite_errors(
+ "ec2",
+ {"region": "us-west-2", "ami_id": "ami-123", "ssh_key_path": ssh_key_path},
+ )
+
+ assert len(errors) == 1
+ assert "ssh_key_path" in errors[0]
+ assert "existing regular file" in errors[0]
+
+
+def test_ec2_environment_kwargs_accept_existing_ssh_key_path(tmp_path: Path) -> None:
+ ssh_key = tmp_path / "id_ed25519"
+ ssh_key.write_text("placeholder", encoding="utf-8")
+
+ assert (
+ _environment_kwarg_prerequisite_errors(
+ "ec2",
+ {"region": "us-west-2", "ami_id": "ami-123", "ssh_key_path": str(ssh_key)},
+ )
+ == []
+ )
+
+
+@pytest.mark.parametrize("subnet_id", [None, ""])
+def test_ec2_private_ephemeral_environment_requires_subnet(subnet_id: str | None) -> None:
+ environment_kwargs: dict[str, object] = {
+ "region": "us-west-2",
+ "ami_id": "ami-123",
+ "use_public_ip": False,
+ }
+ if subnet_id is not None:
+ environment_kwargs["subnet_id"] = subnet_id
+
+ errors = _environment_kwarg_prerequisite_errors("ec2", environment_kwargs)
+
+ assert len(errors) == 1
+ assert "use_public_ip=False requires" in errors[0]
+ assert "subnet_id" in errors[0]
+
+
+def test_ec2_private_ephemeral_environment_accepts_nonempty_subnet() -> None:
+ assert (
+ _environment_kwarg_prerequisite_errors(
+ "ec2",
+ {
+ "region": "us-west-2",
+ "ami_id": "ami-123",
+ "use_public_ip": False,
+ "subnet_id": "subnet-123",
+ },
+ )
+ == []
+ )
+
+
+@pytest.mark.parametrize(
+ ("environment_kwargs", "subprocess_env", "ready"),
+ [
+ ({}, {}, False),
+ ({}, {"OPENSANDBOX_DOMAIN": "sandbox.example.test"}, True),
+ ({"domain": "sandbox.example.test"}, {}, True),
+ ({"domain": None}, {}, False),
+ ({"domain": None}, {"OPENSANDBOX_DOMAIN": "sandbox.example.test"}, True),
+ ({"domain": ""}, {"OPENSANDBOX_DOMAIN": "sandbox.example.test"}, False),
+ ({"domain": " "}, {"OPENSANDBOX_DOMAIN": "sandbox.example.test"}, False),
+ ],
+)
+def test_opensandbox_domain_preflight_uses_effective_child_configuration(
+ monkeypatch: pytest.MonkeyPatch,
+ environment_kwargs: dict[str, object],
+ subprocess_env: dict[str, str],
+ ready: bool,
+) -> None:
+ from harbor.environments.factory import EnvironmentFactory
+
+ monkeypatch.setattr(EnvironmentFactory, "run_preflight", lambda *_args, **_kwargs: None)
+
+ errors = _check_prerequisites(
+ env_mode="opensandbox",
+ agents=[],
+ environment_kwargs=environment_kwargs,
+ subprocess_env=subprocess_env,
+ )
+
+ assert (errors == []) is ready
+ if not ready:
+ assert len(errors) == 1
+ assert "domain" in errors[0]
+
+
+def test_native_environment_required_kwargs_reject_whitespace_padded_ec2_launch_mode() -> None:
+ errors = _environment_kwarg_prerequisite_errors(
+ "ec2",
+ {"region": "us-west-2", "launch_mode": " attach ", "instance_id": "i-123"},
+ )
+
+ assert errors == ["Harbor environment 'ec2' requires launch_mode to be 'ephemeral' or 'attach'"]
+
+
+def test_native_environment_install_hints_use_real_harbor_022_extra_names() -> None:
+ assert "harbor[gke]==0.22.0" in _environment_extra_install_hint("ack")
+ assert "harbor[cloud]==0.22.0" not in _environment_extra_install_hint("ack")
+ assert "harbor[cua]==0.22.0" in _environment_extra_install_hint("cua-cloud")
+ assert "no Python extra" in _environment_extra_install_hint("openshift")
def test_judge_model_overrides_are_forwarded_only_as_harbor_verifier_env() -> None:
@@ -174,9 +661,13 @@ def test_docker_bridge_command_combines_custom_agent_and_secure_environment() ->
agent_import_path=import_path,
)
- assert command[command.index("--agent-import-path") + 1] == import_path
+ assert "--agent-import-path" not in command
+ assert "--environment-import-path" not in command
assert "-a" not in command
- assert "--environment-import-path" in command
+ assert command[command.index("--agent") + 1] == import_path
+ assert command[command.index("--env") + 1] == (
+ "skillevaluator.tier3.harbor.secure_docker_environment:SkillEvaluatorSecureDockerEnvironment"
+ )
def test_local_bridge_command_uses_custom_agent_import_path() -> None:
@@ -190,9 +681,13 @@ def test_local_bridge_command_uses_custom_agent_import_path() -> None:
agent_import_path=import_path,
)
- assert command[command.index("--agent-import-path") + 1] == import_path
- assert "--environment-import-path" in command
+ assert "--agent-import-path" not in command
+ assert "--environment-import-path" not in command
assert "-a" not in command
+ assert command[command.index("--agent") + 1] == import_path
+ assert command[command.index("--env") + 1] == (
+ "skillevaluator.tier3.harbor.local_environment:SkillEvaluatorLocalEnvironment"
+ )
def test_custom_agent_import_path_is_rejected_for_native_cloud() -> None:
@@ -356,6 +851,55 @@ def test_doctor_rejects_alias_model_collision_consistently(monkeypatch) -> None:
assert "specify only one model for claude-code" in normalized
+def test_doctor_ack_preflight_uses_the_exact_resolved_bedrock_environment(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ provider = ProviderConfig(
+ provider="bedrock",
+ model="us.anthropic.claude-test",
+ api_key=None,
+ base_url=None,
+ litellm_model="bedrock/us.anthropic.claude-test",
+ region="us-west-2",
+ )
+ captured: dict[str, object] = {}
+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "eks-exec-auth-key")
+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "eks-exec-auth-secret")
+ monkeypatch.setenv("KUBECONFIG", "/config/eks")
+ monkeypatch.setenv("ALIBABA_CLOUD_ACCESS_KEY_ID", "ambient-parent-only")
+ monkeypatch.setattr(tier3_commands, "resolve_llm_provider", lambda: provider)
+ monkeypatch.setattr(
+ tier3_commands,
+ "_check_prerequisites",
+ lambda **kwargs: captured.update(kwargs) or [],
+ )
+
+ result = CliRunner().invoke(
+ cli,
+ [
+ "doctor",
+ "--agents",
+ "claude-code",
+ "--env-mode",
+ "ack",
+ "--environment-kwarg",
+ "namespace=skill-evals",
+ ],
+ )
+
+ assert result.exit_code == 0, result.output
+ assert captured["env_mode"] == "ack"
+ assert captured["environment_kwargs"] == {"namespace": "skill-evals"}
+ child_env = captured["subprocess_env"]
+ assert isinstance(child_env, dict)
+ assert child_env["KUBECONFIG"] == "/config/eks"
+ assert child_env["AWS_ACCESS_KEY_ID"] == "eks-exec-auth-key"
+ assert child_env["AWS_SECRET_ACCESS_KEY"] == "eks-exec-auth-secret"
+ assert child_env["AWS_REGION"] == "us-west-2"
+ assert child_env["CLAUDE_CODE_USE_BEDROCK"] == "1"
+ assert "ALIBABA_CLOUD_ACCESS_KEY_ID" not in child_env
+
+
def test_generated_task_stages_public_provider_variables_for_the_verifier(tmp_path) -> None:
_write_task_toml(
tmp_path,
diff --git a/tests/tier3/test_evals_config.py b/tests/tier3/test_evals_config.py
index e911debe..6d984b9d 100644
--- a/tests/tier3/test_evals_config.py
+++ b/tests/tier3/test_evals_config.py
@@ -1,10 +1,17 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
+import json
+
import pytest
from skillevaluator.tier3.dataset_utils import load_dataset_entries_with_format
-from skillevaluator.tier3.evals_config import EvalsConfigError, load_evals_config
+from skillevaluator.tier3.evals_config import (
+ EvalsConfigError,
+ load_evals_config,
+ parse_environment_kwarg_overrides,
+ validate_environment_kwargs,
+)
from skillevaluator.tier3.evals_spec import validate_skillevaluators as validate_skill_evals
@@ -62,6 +69,55 @@ def test_load_evals_config_valid_harbor_policy(tmp_path):
assert config["grading"]["mode"] == "default_plus_custom"
+@pytest.mark.parametrize("value", [".nan", ".inf", "-.inf"])
+def test_load_evals_config_rejects_nonfinite_timeout_multiplier(tmp_path, value):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "config.yml").write_text(
+ f"schema_version: 1\nharbor:\n timeout_multiplier: {value}\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(EvalsConfigError, match=r"harbor\.timeout_multiplier.*finite"):
+ load_evals_config(skill)
+
+
+def test_load_evals_config_rejects_overflowing_timeout_multiplier(tmp_path):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "config.yml").write_text(
+ f"schema_version: 1\nharbor:\n timeout_multiplier: {10**1000}\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(EvalsConfigError, match=r"harbor\.timeout_multiplier.*finite"):
+ load_evals_config(skill)
+
+
+def test_load_evals_config_rejects_finite_multiplier_that_overflows_default_timeouts(tmp_path):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "config.yml").write_text(
+ "schema_version: 1\nharbor:\n timeout_multiplier: 1.0e+308\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(EvalsConfigError, match=r"harbor\.timeout_multiplier.*finite Harbor timeouts"):
+ load_evals_config(skill)
+
+
+def test_load_evals_config_rejects_overflowing_pass_threshold(tmp_path):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "config.yml").write_text(
+ f"schema_version: 1\nharbor:\n pass_threshold: {10**1000}\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(EvalsConfigError, match=r"harbor\.pass_threshold.*between"):
+ load_evals_config(skill)
+
+
def test_load_evals_config_missing_is_empty(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
@@ -88,6 +144,149 @@ def test_load_evals_config_rejects_unknown_keys(tmp_path):
load_evals_config(skill)
+def test_load_evals_config_rejects_environment_kwargs_because_they_are_operator_only(tmp_path):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "config.yml").write_text(
+ "schema_version: 1\nharbor:\n environment_kwargs:\n region: us-west-2\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(EvalsConfigError, match=r"unknown harbor key.*environment_kwargs"):
+ load_evals_config(skill)
+
+
+@pytest.mark.parametrize(
+ ("entry", "secret"),
+ [
+ ("api_key=do-not-render-this", "do-not-render-this"),
+ ('headers={"Authorization":"do-not-render-this"}', "do-not-render-this"),
+ ('headers={"X-API-Key":"do-not-render-this"}', "do-not-render-this"),
+ ('metadata={"clientSecret":"do-not-render-this"}', "do-not-render-this"),
+ ('nested={"sudoPassword":"do-not-render-this"}', "do-not-render-this"),
+ ("endpoint=https://bearer-token@example.invalid/path", "bearer-token"),
+ ("endpoint=https://user%3Apassword@example.invalid/path", "user%3Apassword"),
+ ("endpoint=ssh://git@example.invalid/repo", "git@example.invalid"),
+ ("proxy=https://user:do-not-render-this@example.test", "do-not-render-this"),
+ ("missing-equals-do-not-render-this", "do-not-render-this"),
+ ],
+)
+def test_cli_environment_kwargs_reject_secrets_without_echoing_values(entry, secret):
+ with pytest.raises(ValueError) as caught:
+ parse_environment_kwarg_overrides((entry,))
+
+ assert secret not in str(caught.value)
+
+
+def test_cli_environment_kwargs_allow_non_secret_key_names() -> None:
+ assert parse_environment_kwarg_overrides(('key_name="evaluation-key"',)) == {"key_name": "evaluation-key"}
+
+
+def test_cli_environment_kwargs_allow_kubernetes_secret_object_reference() -> None:
+ assert parse_environment_kwarg_overrides(
+ ('image_pull_secret="registry-credentials"',),
+ env_mode="ack",
+ ) == {"image_pull_secret": "registry-credentials"}
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "reference"),
+ [
+ ("modal", "registry-credentials"),
+ ("daytona", "registry-credentials"),
+ ("ack", "username:password"),
+ ("ack", "UPPER_CASE"),
+ ("ack", "contains spaces"),
+ ],
+)
+def test_cli_environment_kwargs_reject_image_pull_secret_outside_ack_or_invalid_kubernetes_names(
+ env_mode: str,
+ reference: str,
+) -> None:
+ with pytest.raises(ValueError, match="Invalid --environment-kwarg"):
+ parse_environment_kwarg_overrides(
+ (f"image_pull_secret={json.dumps(reference)}",),
+ env_mode=env_mode,
+ )
+
+
+def test_environment_kwargs_reject_cycles_without_recursing() -> None:
+ cyclic: dict[str, object] = {}
+ cyclic["nested"] = cyclic
+
+ with pytest.raises(ValueError, match="cyclic"):
+ validate_environment_kwargs({"options": cyclic})
+
+
+def test_environment_kwargs_reject_excessive_direct_api_nesting_without_recursing() -> None:
+ nested: object = "leaf"
+ for _ in range(40):
+ nested = [nested]
+
+ with pytest.raises(ValueError, match="nest at most"):
+ validate_environment_kwargs({"options": nested})
+
+
+def test_cli_environment_kwargs_reject_extreme_json_nesting_without_traceback() -> None:
+ deeply_nested = "[" * 1200 + "0" + "]" * 1200
+
+ with pytest.raises(ValueError, match="nest"):
+ parse_environment_kwarg_overrides((f"options={deeply_nested}",))
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "entry", "expected"),
+ [
+ (
+ "daytona",
+ 'secrets={"TARGET_API_KEY":"organization-secret-name"}',
+ {"secrets": {"TARGET_API_KEY": "organization-secret-name"}},
+ ),
+ (
+ "modal",
+ 'secrets=["runtime-secret","telemetry-secret"]',
+ {"secrets": ["runtime-secret", "telemetry-secret"]},
+ ),
+ (
+ "modal",
+ 'registry_secret="private-registry-login"',
+ {"registry_secret": "private-registry-login"},
+ ),
+ (
+ "skypilot",
+ 'secrets=["cluster-secret"]',
+ {"secrets": ["cluster-secret"]},
+ ),
+ (
+ "cwsandbox",
+ 'secrets=[{"store":"team-store","name":"runtime-key","field":"value","env_var":"API_KEY"}]',
+ {"secrets": [{"store": "team-store", "name": "runtime-key", "field": "value", "env_var": "API_KEY"}]},
+ ),
+ (
+ "wandb",
+ 'secrets=[{"name":"runtime-key","env_var":"API_KEY"}]',
+ {"secrets": [{"name": "runtime-key", "env_var": "API_KEY"}]},
+ ),
+ ],
+)
+def test_cli_environment_kwargs_allow_provider_secret_references(env_mode, entry, expected) -> None:
+ assert parse_environment_kwarg_overrides((entry,), env_mode=env_mode) == expected
+
+
+@pytest.mark.parametrize(
+ ("env_mode", "entry"),
+ [
+ ("e2b", 'secrets=["not-supported"]'),
+ ("daytona", 'secrets=["wrong-shape"]'),
+ ("modal", 'secrets={"wrong":"shape"}'),
+ ("cwsandbox", 'secrets=[{"value":"plaintext-not-a-reference"}]'),
+ ],
+)
+def test_cli_environment_kwargs_reject_secret_reference_fields_outside_exact_harbor_shapes(env_mode, entry) -> None:
+ with pytest.raises(ValueError, match="Invalid --environment-kwarg"):
+ parse_environment_kwarg_overrides((entry,), env_mode=env_mode)
+
+
def test_load_evals_config_validates_resource_shapes(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
diff --git a/tests/tier3/test_harbor_local_agents.py b/tests/tier3/test_harbor_local_agents.py
index 1b68eb47..9c1f08a1 100644
--- a/tests/tier3/test_harbor_local_agents.py
+++ b/tests/tier3/test_harbor_local_agents.py
@@ -2,21 +2,150 @@
# SPDX-License-Identifier: Apache-2.0
import asyncio
+import contextlib
+import os
+import tomllib
+from pathlib import Path
+from types import SimpleNamespace
import pytest
pytest.importorskip("harbor")
+from harbor.agents.installed.base import NonZeroAgentExitCodeError
from harbor.agents.installed.codex import Codex
-from harbor.models.trial.paths import EnvironmentPaths
+from harbor.environments.base import BaseEnvironment, ExecResult
+from harbor.models.task.config import EnvironmentConfig, MCPServerConfig
+from harbor.models.trial.paths import EnvironmentPaths, TrialPaths
from skillevaluator.tier3.harbor.local_agents import (
SkillEvaluatorLocalClaudeCode,
SkillEvaluatorLocalCodex,
SkillEvaluatorLocalOpenCode,
+ SkillEvaluatorNvidiaBuildClaudeCode,
+ SkillEvaluatorNvidiaBuildCodex,
)
+class _RecordingEnvironment:
+ default_user = None
+
+ def __init__(self) -> None:
+ self.uploads: list[tuple[str, str]] = []
+
+ async def upload_file(self, source: object, destination: object) -> None:
+ self.uploads.append((str(destination), Path(source).read_text(encoding="utf-8")))
+
+ @contextlib.contextmanager
+ def scoped_exec_env(self, _env: dict[str, str]):
+ yield
+
+ def last_codex_config(self, remote_home: object) -> dict[str, object]:
+ remote_path = f"{remote_home}/config.toml"
+ configs = [content for destination, content in self.uploads if destination == remote_path]
+ assert configs, f"no Codex config was uploaded to {remote_path}"
+ return tomllib.loads(configs[-1])
+
+
+class _MergingRecordingEnvironment(BaseEnvironment):
+ """Real Harbor env scoping with a local child process for final-env proof."""
+
+ def __init__(self, tmp_path: Path) -> None:
+ self.commands: list[str] = []
+ self.merged_exec_envs: list[dict[str, str]] = []
+ self.actual_child_envs: list[dict[str, str]] = []
+ super().__init__(
+ environment_dir=tmp_path,
+ environment_name="recording",
+ session_id="recording-session",
+ trial_paths=TrialPaths(tmp_path / "trial"),
+ task_env_config=EnvironmentConfig(),
+ persistent_env={"PERSISTENT_VALUE": "persistent"},
+ )
+
+ @staticmethod
+ def type() -> str:
+ return "recording"
+
+ def _validate_definition(self) -> None:
+ return None
+
+ async def start(self, force_build: bool) -> None:
+ _ = force_build
+
+ async def stop(self, delete: bool) -> None:
+ _ = delete
+
+ async def upload_file(self, source_path: Path | str, target_path: str) -> None:
+ _ = (source_path, target_path)
+
+ async def upload_dir(self, source_dir: Path | str, target_dir: str) -> None:
+ _ = (source_dir, target_dir)
+
+ async def download_file(self, source_path: str, target_path: Path | str) -> None:
+ _ = (source_path, target_path)
+
+ async def download_dir(self, source_dir: str, target_dir: Path | str) -> None:
+ _ = (source_dir, target_dir)
+
+ async def exec(
+ self,
+ command: str,
+ cwd: str | None = None,
+ env: dict[str, str] | None = None,
+ timeout_sec: int | None = None,
+ user: str | int | None = None,
+ ) -> ExecResult:
+ _ = (timeout_sec, user)
+ merged = self._merge_env(env) or {}
+ self.commands.append(command)
+ self.merged_exec_envs.append(dict(merged))
+
+ process_env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), **merged}
+ process = await asyncio.create_subprocess_exec(
+ "/bin/bash",
+ "-c",
+ command,
+ cwd=cwd,
+ env=process_env,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ stdout_bytes, stderr_bytes = await process.communicate()
+ stdout = stdout_bytes.decode()
+ child_env: dict[str, str] = {}
+ for line in stdout.splitlines():
+ key, separator, value = line.partition("=")
+ if separator:
+ child_env[key] = value
+ self.actual_child_envs.append(child_env)
+ return ExecResult(
+ stdout=stdout,
+ stderr=stderr_bytes.decode(),
+ return_code=process.returncode,
+ )
+
+
+def _record_installed_agent_exec(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, dict[str, str]]]:
+ calls: list[tuple[str, dict[str, str]]] = []
+
+ async def fake_parent_exec(
+ _self: object,
+ _environment: object,
+ command: str,
+ env: dict[str, str] | None = None,
+ **_kwargs: object,
+ ) -> SimpleNamespace:
+ calls.append((command, dict(env or {})))
+ return SimpleNamespace(return_code=0, stdout="", stderr="")
+
+ monkeypatch.setattr(
+ "harbor.agents.installed.base.BaseInstalledAgent.exec_as_agent",
+ fake_parent_exec,
+ )
+ return calls
+
+
def test_local_codex_uses_per_trial_codex_home() -> None:
assert str(SkillEvaluatorLocalCodex._REMOTE_CODEX_HOME).startswith(EnvironmentPaths.agent_dir.as_posix())
assert str(SkillEvaluatorLocalCodex._REMOTE_CODEX_SECRETS_DIR).startswith(EnvironmentPaths.agent_dir.as_posix())
@@ -24,24 +153,144 @@ def test_local_codex_uses_per_trial_codex_home() -> None:
assert str(SkillEvaluatorLocalCodex._REMOTE_CODEX_SECRETS_DIR) != "/tmp/codex-secrets"
-def test_local_codex_creates_per_trial_state_dirs(monkeypatch, tmp_path) -> None:
- commands: list[str] = []
+def test_local_codex_uploads_openai_responses_config_and_creates_per_trial_state_dirs(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ calls = _record_installed_agent_exec(monkeypatch)
+ monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+ monkeypatch.delenv("OPENAI_API_BASE", raising=False)
+ environment = _RecordingEnvironment()
+ agent = SkillEvaluatorLocalCodex(
+ logs_dir=tmp_path,
+ model_name="openai/gpt-5.4",
+ extra_env={"OPENAI_API_KEY": "test-key"},
+ )
- async def fake_exec_as_agent(_environment, command, **_kwargs):
- commands.append(command)
+ asyncio.run(agent.run("do the thing", environment=environment, context=object()))
- async def fake_parent_run(self, instruction, environment, context):
- return None
+ setup_command, setup_env = calls[0]
+ assert 'mkdir -p "$CODEX_HOME"' in setup_command
+ assert SkillEvaluatorLocalCodex._REMOTE_CODEX_SECRETS_DIR.as_posix() in setup_command
+ assert setup_env["CODEX_HOME"] == SkillEvaluatorLocalCodex._REMOTE_CODEX_HOME.as_posix()
+ config = environment.last_codex_config(SkillEvaluatorLocalCodex._REMOTE_CODEX_HOME)
+ assert config["model_provider"] == "openai_compatible"
+ assert config["model_providers"] == {
+ "openai_compatible": {
+ "name": "OpenAI-compatible provider",
+ "base_url": "https://api.openai.com/v1",
+ "env_key": "OPENAI_API_KEY",
+ "wire_api": "responses",
+ }
+ }
+
+
+def test_local_codex_final_config_and_launcher_preserve_explicit_gateway_routing(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ calls = _record_installed_agent_exec(monkeypatch)
+ environment = _RecordingEnvironment()
+ gateway_url = "https://gateway.example/v1"
+ monkeypatch.setenv("OPENAI_BASE_URL", gateway_url)
+ model_name = "openai/openai/gpt-5.4"
+ agent = SkillEvaluatorLocalCodex(
+ logs_dir=tmp_path,
+ model_name=model_name,
+ extra_env={"OPENAI_API_KEY": "test-key"},
+ )
- agent = SkillEvaluatorLocalCodex(logs_dir=tmp_path, model_name="openai/test")
- monkeypatch.setattr(agent, "exec_as_agent", fake_exec_as_agent)
- monkeypatch.setattr(Codex, "run", fake_parent_run)
+ asyncio.run(agent.run("do the thing", environment=environment, context=object()))
+
+ config = environment.last_codex_config(SkillEvaluatorLocalCodex._REMOTE_CODEX_HOME)
+ assert config["model_provider"] == "openai_compatible"
+ assert config["openai_base_url"] == gateway_url
+ assert config["model_providers"]["openai_compatible"]["base_url"] == gateway_url
+ run_command = next(command for command, _env in calls if "codex exec" in command)
+ assert f"--model {model_name} " in run_command
+ assert "--model gpt-5.4 " not in run_command
+
+
+def test_local_codex_final_config_preserves_user_and_harbor_mcp_servers(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ _record_installed_agent_exec(monkeypatch)
+ monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+ monkeypatch.delenv("OPENAI_API_BASE", raising=False)
+ environment = _RecordingEnvironment()
+ agent = SkillEvaluatorLocalCodex(
+ logs_dir=tmp_path,
+ model_name="openai/gpt-5.4",
+ extra_env={"OPENAI_API_KEY": "test-key"},
+ config={
+ "mcp_servers": {
+ "user-tools": {"url": "https://user-tools.example/mcp"},
+ }
+ },
+ mcp_servers=[
+ MCPServerConfig(
+ name="task-tools",
+ transport="stdio",
+ command="python3",
+ args=["-m", "task_tools"],
+ )
+ ],
+ )
- asyncio.run(agent.run("do the thing", environment=object(), context=object()))
+ asyncio.run(agent.run("do the thing", environment=environment, context=object()))
+
+ config = environment.last_codex_config(SkillEvaluatorLocalCodex._REMOTE_CODEX_HOME)
+ assert config["mcp_servers"] == {
+ "user-tools": {"url": "https://user-tools.example/mcp"},
+ "task-tools": {"command": "python3", "args": ["-m", "task_tools"]},
+ }
+ assert config["model_provider"] == "openai_compatible"
+
+
+def test_local_codex_runtime_provider_fields_win_without_losing_unrelated_user_config(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ _record_installed_agent_exec(monkeypatch)
+ environment = _RecordingEnvironment()
+ gateway_url = "https://runtime.example/v1"
+ monkeypatch.setenv("OPENAI_BASE_URL", "https://ambient-must-not-win.example/v1")
+ agent = SkillEvaluatorLocalCodex(
+ logs_dir=tmp_path,
+ model_name="openai/gpt-5.4",
+ extra_env={"OPENAI_API_KEY": "test-key", "OPENAI_BASE_URL": gateway_url},
+ config={
+ "approval_policy": "never",
+ "model_provider": "user-provider",
+ "openai_base_url": "https://user.example/v1",
+ "model_providers": {
+ "openai_compatible": {
+ "name": "User provider",
+ "base_url": "https://user.example/v1",
+ "env_key": "USER_API_KEY",
+ "wire_api": "chat_completions",
+ "http_headers": {"X-User": "preserved"},
+ },
+ "user-provider": {"base_url": "https://other.example/v1"},
+ },
+ },
+ )
- setup_command = commands[0]
- assert f"mkdir -p {SkillEvaluatorLocalCodex._REMOTE_CODEX_HOME.as_posix()}" in setup_command
- assert SkillEvaluatorLocalCodex._REMOTE_CODEX_SECRETS_DIR.as_posix() in setup_command
+ asyncio.run(agent.run("do the thing", environment=environment, context=object()))
+
+ config = environment.last_codex_config(SkillEvaluatorLocalCodex._REMOTE_CODEX_HOME)
+ assert config["approval_policy"] == "never"
+ assert config["model_provider"] == "openai_compatible"
+ assert config["openai_base_url"] == gateway_url
+ assert config["model_providers"]["user-provider"] == {"base_url": "https://other.example/v1"}
+ assert config["model_providers"]["openai_compatible"] == {
+ "name": "OpenAI-compatible provider",
+ "base_url": gateway_url,
+ "env_key": "OPENAI_API_KEY",
+ "wire_api": "responses",
+ "http_headers": {"X-User": "preserved"},
+ }
def test_local_codex_rewrites_upstream_tmp_secrets_dir(monkeypatch, tmp_path) -> None:
@@ -92,10 +341,10 @@ async def fake_parent_exec(self, environment, command, **kwargs):
def test_local_claude_does_not_rewrite_instruction_permission_text(monkeypatch, tmp_path) -> None:
- commands: list[str] = []
+ calls: list[tuple[str, dict[str, str]]] = []
- async def fake_parent_exec(self, environment, command, **kwargs):
- commands.append(command)
+ async def fake_parent_exec(self, environment, command, env=None, **kwargs):
+ calls.append((command, dict(env or {})))
monkeypatch.setattr(
"harbor.agents.installed.base.BaseInstalledAgent.exec_as_agent",
@@ -104,20 +353,230 @@ async def fake_parent_exec(self, environment, command, **kwargs):
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test")
monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://provider.example/v1")
+ instruction = "quote --permission-mode=bypassPermissions literally\nwithout rewriting 'bytes'"
agent = SkillEvaluatorLocalClaudeCode(logs_dir=tmp_path, model_name="aws/anthropic/bedrock-claude-opus-4-6")
asyncio.run(
agent.run(
- "quote --permission-mode=bypassPermissions literally",
+ instruction,
environment=object(),
context=object(),
)
)
- run_command = next(command for command in commands if "claude --verbose" in command)
- launcher, _separator, prompt = run_command.partition(" -- ")
- assert "--permission-mode=auto" in launcher
- assert "--permission-mode=bypassPermissions" not in launcher
- assert "--permission-mode=bypassPermissions" in prompt
+ run_command, run_env = next((command, env) for command, env in calls if "claude --verbose" in command)
+ instruction_vars = {
+ key: value for key, value in run_env.items() if key.startswith("HARBOR_") and "_INSTRUCTION_" in key
+ }
+ assert instruction_vars and list(instruction_vars.values()) == [instruction]
+ assert next(iter(instruction_vars.values())).encode() == instruction.encode()
+ assert "--permission-mode=auto" in run_command
+ assert "--permission-mode=bypassPermissions" not in run_command
+ assert instruction not in run_command
+
+
+def test_nvidia_build_codex_final_config_keeps_dynamic_bridge_over_user_and_runtime_values(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ calls = _record_installed_agent_exec(monkeypatch)
+ environment = _RecordingEnvironment()
+ bridge_origin = "http://127.0.0.1:43123"
+ agent = SkillEvaluatorNvidiaBuildCodex(
+ logs_dir=tmp_path,
+ model_name="nvidia/meta/llama-3.1-8b-instruct",
+ extra_env={
+ "OPENAI_API_KEY": "external-key",
+ "OPENAI_BASE_URL": "https://runtime-must-not-win.example/v1",
+ },
+ config={
+ "approval_policy": "never",
+ "openai_base_url": "https://user-must-not-win.example/v1",
+ "model_provider": "user-provider",
+ "model_providers": {
+ "openai_compatible": {
+ "base_url": "https://user-must-not-win.example/v1",
+ "http_headers": {"X-User": "preserved"},
+ }
+ },
+ "mcp_servers": {"user-tools": {"url": "https://user-tools.example/mcp"}},
+ },
+ )
+
+ async def fake_start_bridge(_environment: object) -> None:
+ agent._nvidia_build_bridge_started = True
+ agent._nvidia_build_bridge_origin = bridge_origin
+ agent._nvidia_build_bridge_client_token = "bridge-client-token"
+
+ async def fake_cleanup_bridge(_environment: object) -> None:
+ agent._nvidia_build_bridge_started = False
+ agent._nvidia_build_bridge_origin = None
+ agent._nvidia_build_bridge_client_token = None
+
+ monkeypatch.setattr(agent, "_start_bridge", fake_start_bridge)
+ monkeypatch.setattr(agent, "_cleanup_bridge", fake_cleanup_bridge)
+
+ asyncio.run(agent.run("do the thing", environment=environment, context=object()))
+
+ config = environment.last_codex_config(Codex._REMOTE_CODEX_HOME)
+ assert config["approval_policy"] == "never"
+ assert config["mcp_servers"] == {"user-tools": {"url": "https://user-tools.example/mcp"}}
+ assert config["model_provider"] == "openai_compatible"
+ assert config["openai_base_url"] == f"{bridge_origin}/v1"
+ assert config["model_providers"]["openai_compatible"] == {
+ "name": "OpenAI-compatible provider",
+ "base_url": f"{bridge_origin}/v1",
+ "env_key": "OPENAI_API_KEY",
+ "wire_api": "responses",
+ "http_headers": {"X-User": "preserved"},
+ }
+ run_command, run_env = next((command, env) for command, env in calls if "codex exec" in command)
+ assert "--model nvidia/meta/llama-3.1-8b-instruct " in run_command
+ assert run_env["OPENAI_API_KEY"] == "bridge-client-token"
+ assert "OPENAI_BASE_URL" not in run_env
+ assert "external-key" not in run_command
+
+
+def test_nvidia_build_codex_get_env_delegates_alternatives_and_protects_bridge_values(
+ tmp_path: Path,
+) -> None:
+ agent = SkillEvaluatorNvidiaBuildCodex(
+ logs_dir=tmp_path,
+ model_name="nvidia/meta/llama-3.1-8b-instruct",
+ extra_env={
+ "FALLBACK_TOKEN": "fallback-value",
+ "OPENAI_API_KEY": "external-key",
+ "OPENAI_BASE_URL": "https://external.example/v1",
+ },
+ )
+
+ assert agent._get_env("PRIMARY_TOKEN", "FALLBACK_TOKEN") == "fallback-value"
+
+ agent._nvidia_build_bridge_client_env = {"OPENAI_API_KEY": "bridge-client-token"}
+ assert agent._get_env("PRIMARY_TOKEN", "FALLBACK_TOKEN") == "fallback-value"
+ assert agent._get_env("PRIMARY_TOKEN", "OPENAI_API_KEY") == "bridge-client-token"
+ assert agent._get_env("OPENAI_BASE_URL", "OPENAI_API_BASE") is None
+
+
+def test_nvidia_build_codex_bridge_scope_wins_real_harbor_env_merge_and_resets(
+ tmp_path: Path,
+) -> None:
+ environment = _MergingRecordingEnvironment(tmp_path)
+ external_values = {
+ "OPENAI_API_KEY": "external-openai-key",
+ "OPENAI_BASE_URL": "https://external-base.example/v1",
+ "OPENAI_API_BASE": "https://external-alias.example/v1",
+ "NVIDIA_API_KEY": "external-nvidia-key",
+ "OUTER_ONLY": "outer-value",
+ }
+ agent = SkillEvaluatorNvidiaBuildCodex(
+ logs_dir=tmp_path,
+ model_name="nvidia/meta/llama-3.1-8b-instruct",
+ extra_env=external_values,
+ )
+ agent._nvidia_build_bridge_origin = "http://127.0.0.1:43123"
+ agent._nvidia_build_bridge_client_env = {"OPENAI_API_KEY": "bridge-client-token"}
+ snapshots: dict[str, dict[str, str] | None] = {}
+
+ async def exercise() -> None:
+ snapshots["before"] = environment._merge_env(None)
+ with environment.scoped_exec_env(agent.extra_env):
+ snapshots["outer_before"] = environment._merge_env(None)
+ await agent.exec_as_agent(environment, command="env")
+ snapshots["outer_after"] = environment._merge_env(None)
+ snapshots["after"] = environment._merge_env(None)
+
+ asyncio.run(exercise())
+
+ assert environment.merged_exec_envs[-1]["OPENAI_API_KEY"] == "bridge-client-token"
+ child_env = environment.actual_child_envs[-1]
+ assert child_env["OPENAI_API_KEY"] == "bridge-client-token"
+ assert child_env["OUTER_ONLY"] == "outer-value"
+ for name in ("OPENAI_BASE_URL", "OPENAI_API_BASE", "NVIDIA_API_KEY"):
+ assert name not in child_env
+ external_secret_values = {
+ external_values[name] for name in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE", "NVIDIA_API_KEY")
+ }
+ assert not external_secret_values.intersection(child_env.values())
+
+ command = environment.commands[-1]
+ assert "env -u NVIDIA_API_KEY -u OPENAI_BASE_URL -u OPENAI_API_BASE bash -o pipefail -c env" in command
+ assert "bridge-client-token" not in command
+ assert not any(value in command for value in external_values.values())
+
+ assert snapshots["before"] == {"PERSISTENT_VALUE": "persistent"}
+ assert snapshots["outer_before"] == {"PERSISTENT_VALUE": "persistent", **external_values}
+ assert snapshots["outer_after"] == snapshots["outer_before"]
+ assert snapshots["after"] == snapshots["before"]
+ assert environment._exec_env_overlays.get() == ()
+
+
+def test_nvidia_build_claude_client_scope_wins_outer_agent_env(
+ tmp_path: Path,
+) -> None:
+ environment = _MergingRecordingEnvironment(tmp_path)
+ agent = SkillEvaluatorNvidiaBuildClaudeCode(
+ logs_dir=tmp_path,
+ model_name="nvidia/nemotron-3-super-120b-a12b",
+ extra_env={
+ "ANTHROPIC_API_KEY": "external-anthropic-key",
+ "ANTHROPIC_BASE_URL": "https://external-anthropic.example",
+ "ANTHROPIC_MODEL": "external-model",
+ "NVIDIA_API_KEY": "external-nvidia-key",
+ },
+ )
+ agent._nvidia_build_bridge_client_token = "bridge-client-token"
+ agent._nvidia_build_bridge_origin = "http://127.0.0.1:43123"
+ client_env = agent._bridge_client_environment()
+ agent._nvidia_build_bridge_client_env = client_env
+
+ async def exercise() -> None:
+ with environment.scoped_exec_env(agent.extra_env):
+ await agent.exec_as_agent(environment, command="env")
+
+ asyncio.run(exercise())
+
+ child_env = environment.actual_child_envs[-1]
+ for name, value in client_env.items():
+ assert child_env[name] == value
+ assert "NVIDIA_API_KEY" not in child_env
+ assert "external-anthropic-key" not in child_env.values()
+ assert "external-model" not in child_env.values()
+ assert "bridge-client-token" not in environment.commands[-1]
+
+
+@pytest.mark.parametrize(
+ ("agent_type", "client_env"),
+ (
+ (SkillEvaluatorNvidiaBuildCodex, {"OPENAI_API_KEY": "bridge-client-token"}),
+ (
+ SkillEvaluatorNvidiaBuildClaudeCode,
+ {
+ "ANTHROPIC_API_KEY": "bridge-client-token",
+ "ANTHROPIC_BASE_URL": "http://127.0.0.1:43123",
+ "ANTHROPIC_MODEL": "nvidia/model",
+ },
+ ),
+ ),
+)
+def test_nvidia_build_bridge_preserves_pipeline_failure_status(
+ tmp_path: Path,
+ agent_type: type[SkillEvaluatorNvidiaBuildCodex] | type[SkillEvaluatorNvidiaBuildClaudeCode],
+ client_env: dict[str, str],
+) -> None:
+ environment = _MergingRecordingEnvironment(tmp_path)
+ agent = agent_type(logs_dir=tmp_path, model_name="nvidia/model")
+ agent._nvidia_build_bridge_origin = "http://127.0.0.1:43123"
+ agent._nvidia_build_bridge_client_env = client_env
+
+ with pytest.raises(NonZeroAgentExitCodeError):
+ asyncio.run(
+ agent.exec_as_agent(
+ environment,
+ command="false | tee /dev/null",
+ )
+ )
+
+ assert "bash -o pipefail -c" in environment.commands[-1]
def test_local_codex_preserves_full_gateway_model_name(monkeypatch, tmp_path) -> None:
@@ -223,6 +682,50 @@ async def fake_parent_exec(self, environment, command, **kwargs):
assert any(env.get("OPENAI_BASE_URL") == "https://provider.example/v1" for env in envs)
+def test_local_opencode_nvidia_error_event_raises(monkeypatch, tmp_path) -> None:
+ async def fake_exec_as_agent(_environment, command, **_kwargs):
+ if "opencode run" in command:
+ (tmp_path / "opencode.txt").write_text(
+ '{"type":"error","error":{"data":{"message":"temporary backend failure"}}}\n',
+ encoding="utf-8",
+ )
+
+ agent = SkillEvaluatorLocalOpenCode(logs_dir=tmp_path, model_name="nvidia/test")
+ monkeypatch.setattr(agent, "exec_as_agent", fake_exec_as_agent)
+
+ with pytest.raises(
+ NonZeroAgentExitCodeError,
+ match=r"OpenCode emitted error event\(s\): temporary backend failure",
+ ):
+ asyncio.run(agent.run("do the thing", environment=object(), context=object()))
+
+
+def test_local_opencode_nvidia_trajectory_falls_back_to_instruction_when_user_event_is_missing(
+ monkeypatch,
+ tmp_path,
+) -> None:
+ async def fake_exec_as_agent(_environment, command, **_kwargs):
+ _ = command
+
+ agent = SkillEvaluatorLocalOpenCode(logs_dir=tmp_path, model_name="nvidia/test")
+ monkeypatch.setattr(agent, "exec_as_agent", fake_exec_as_agent)
+
+ asyncio.run(agent.run("do the thing", environment=object(), context=object()))
+ trajectory = agent._convert_events_to_trajectory(
+ [
+ {"type": "step_start", "sessionID": "session-1", "timestamp": 1},
+ {"type": "text", "part": {"type": "text", "text": "done"}},
+ {"type": "step_finish", "part": {"tokens": {}, "cost": 0}},
+ ]
+ )
+
+ assert trajectory is not None
+ assert [(step.source, step.message) for step in trajectory.steps] == [
+ ("user", "do the thing"),
+ ("agent", "done"),
+ ]
+
+
def test_local_opencode_non_nvidia_fallback_renders_prompt_once(monkeypatch, tmp_path) -> None:
commands: list[str] = []
render_count = 0
diff --git a/tests/tier3/test_harbor_security_attribution.py b/tests/tier3/test_harbor_security_attribution.py
index da1ebd56..be9b5046 100644
--- a/tests/tier3/test_harbor_security_attribution.py
+++ b/tests/tier3/test_harbor_security_attribution.py
@@ -1,7 +1,10 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-from skillevaluator.tier3.harbor.collector import _annotate_security_attribution
+from skillevaluator.tier3.harbor.collector import (
+ _annotate_security_attribution,
+ _reward_publication_projection_is_safe,
+)
def _reward(entry_id, findings):
@@ -37,6 +40,7 @@ def test_security_attribution_marks_with_skill_only_after_skill_use_as_skill_rel
finding = with_rewards[0]["details"]["security"]["findings"][0]
assert finding["attribution"] == "likely_skill_related"
+ assert "target skill" in finding["attribution_explanation"]
assert summary["likely_skill_related"] == 1
@@ -83,3 +87,81 @@ def test_security_attribution_avoids_skill_blame_without_baseline():
finding = with_rewards[0]["details"]["security"]["findings"][0]
assert finding["attribution"] == "unknown_no_baseline"
assert summary["unknown_no_baseline"] == 1
+
+
+def test_security_attribution_omits_per_finding_expansion_when_projection_would_overflow():
+ findings = [{"score_impact": True} for _ in range(17_000)]
+ with_rewards = [_reward("case-1", findings)]
+
+ summary = _annotate_security_attribution(with_rewards, [], baseline_run=False)
+
+ security = with_rewards[0]["details"]["security"]
+ assert summary["unknown_no_baseline"] == len(findings)
+ assert security["attribution"] == "unknown_no_baseline"
+ assert "omitted" in security["attribution_completeness"].casefold()
+ assert all("attribution" not in finding for finding in findings)
+
+
+def test_security_attribution_retains_labels_when_only_repeated_explanations_overflow():
+ findings = [{"score_impact": True} for _ in range(9_000)]
+ with_rewards = [_reward("case-1", findings)]
+
+ _annotate_security_attribution(with_rewards, [], baseline_run=False)
+
+ security = with_rewards[0]["details"]["security"]
+ assert "repeated explanations were omitted" in security["attribution_completeness"]
+ projected_findings = security["findings"]
+ assert all(finding["attribution"] == "unknown_no_baseline" for finding in projected_findings)
+ assert all("attribution_explanation" not in finding for finding in projected_findings)
+ assert _reward_publication_projection_is_safe(with_rewards[0])
+
+
+def test_security_improvement_normalizes_malformed_findings_without_crashing():
+ with_rewards = [
+ {
+ "entry_id": "case-1",
+ "details": {"security": {"score": 1.0, "findings": {}}},
+ }
+ ]
+ without_rewards = [_reward("case-1", [_finding()])]
+
+ summary = _annotate_security_attribution(with_rewards, without_rewards)
+
+ security = with_rewards[0]["details"]["security"]
+ assert summary["skill_may_have_improved_safety"] == 1
+ assert isinstance(security["findings"], list)
+ assert security["findings"][0]["type"] == "skill_reduced_unsafe_behavior"
+
+
+def test_security_improvement_does_not_copy_unbounded_baseline_messages():
+ padding = "x" * 1_500_000
+ with_reward = {
+ "entry_id": "case-1",
+ "metric_set": "custom-only",
+ "overall": 1.0,
+ "details": {"padding": padding, "security": {"score": 1.0, "findings": []}},
+ }
+ baseline_finding = _finding()
+ baseline_finding["message"] = padding
+
+ summary = _annotate_security_attribution(
+ [with_reward],
+ [_reward("case-1", [baseline_finding])],
+ )
+
+ security = with_reward["details"]["security"]
+ assert summary["skill_may_have_improved_safety"] == 1
+ assert security["findings"][0]["evidence"].startswith("Without-skill baseline contained 1")
+ assert padding not in security["findings"][0]["evidence"]
+ assert _reward_publication_projection_is_safe(with_reward)
+
+
+def test_security_attribution_bounds_case_details_with_exact_metadata():
+ with_rewards = [_reward(f"case-{index:04d}", []) for index in range(300)]
+
+ summary = _annotate_security_attribution(with_rewards, [], baseline_run=False)
+
+ assert summary["case_details_total"] == 300
+ assert summary["case_details_shown"] == 256
+ assert summary["case_details_truncated"] is True
+ assert len(summary["cases"]) == 256
diff --git a/tests/tier3/test_judge_failure_artifacts.py b/tests/tier3/test_judge_failure_artifacts.py
index e38d1775..5e5071e2 100644
--- a/tests/tier3/test_judge_failure_artifacts.py
+++ b/tests/tier3/test_judge_failure_artifacts.py
@@ -17,10 +17,14 @@
import pytest
-from skillevaluator.tier3.harbor.adapter import _write_test_sh
+from skillevaluator.tier3.harbor.adapter import _EVALUATOR_TESTS_SUBDIR, _write_test_sh
from skillevaluator.tier3.harbor.metrics import (
DEFAULT_METRIC_SET,
+ MAX_CUSTOM_METRIC_NAME_BYTES,
+ MAX_CUSTOM_METRICS,
RESERVED_METRIC_NAMES,
+ custom_metric_name_is_publishable,
+ extract_custom_metrics,
metric_set_for_reward,
overall_score,
)
@@ -283,7 +287,7 @@ def test_evaluation_failure_fields_are_reserved_metadata() -> None:
expected = {"evaluation_status", "evaluation_errors"}
assert expected <= RESERVED_METRIC_NAMES
- assert expected <= custom_grader_runner.RESERVED
+ assert custom_grader_runner.RESERVED == RESERVED_METRIC_NAMES
assert custom_grader_runner._extract_custom_metrics(
{"evaluation_status": 1.0, "evaluation_errors": 0.5, "domain_score": 0.75}
) == {"domain_score": 0.75}
@@ -291,6 +295,193 @@ def test_evaluation_failure_fields_are_reserved_metadata() -> None:
custom_grader_runner._extract_custom_metrics({"custom_metrics": {"evaluation_status": 0.5}})
+@pytest.mark.parametrize("malformed", [None, 0.5, "quality", [0.5]])
+def test_custom_grader_runner_rejects_malformed_custom_metrics_container(malformed: object) -> None:
+ with pytest.raises(RuntimeError, match=r"container.*JSON object"):
+ custom_grader_runner._extract_custom_metrics({"custom_metrics": malformed, "quality": 0.8, "overall": 0.8})
+
+
+@pytest.mark.parametrize(
+ "invalid_score",
+ [math.nan, math.inf, -math.inf, -0.01, 1.01, 1e308, 10**400],
+ ids=[
+ "nan",
+ "positive-infinity",
+ "negative-infinity",
+ "negative",
+ "above-one",
+ "huge-finite",
+ "unrepresentable-integer",
+ ],
+)
+def test_custom_grader_runner_rejects_invalid_overall_and_custom_scores(
+ invalid_score: float | int,
+) -> None:
+ assert custom_grader_runner._score_from_reward({"overall": invalid_score}) is None
+ assert custom_grader_runner._extract_custom_metrics(
+ {"custom_metrics": {"invalid": invalid_score, "valid": 0.5}}
+ ) == {"valid": 0.5}
+ assert custom_grader_runner._numeric_reward_payload(
+ {"invalid": invalid_score, "valid": 0.5},
+ overall=invalid_score,
+ ) == {"valid": 0.5}
+
+
+def test_custom_grader_runner_enforces_metric_publication_contract_before_artifacts() -> None:
+ assert custom_grader_runner.MAX_CUSTOM_METRICS == MAX_CUSTOM_METRICS
+ assert custom_grader_runner.MAX_CUSTOM_METRIC_NAME_BYTES == MAX_CUSTOM_METRIC_NAME_BYTES
+ assert custom_grader_runner._extract_custom_metrics(
+ {
+ "custom_metrics": {
+ "quality": 0.8,
+ "sk-abcdefghijk": 0.7,
+ "quality_sk-abcdefghijk": 0.6,
+ "api_key_quality": 0.5,
+ "secret_handling": 0.9,
+ }
+ }
+ ) == {"quality": 0.8, "secret_handling": 0.9}
+
+ with pytest.raises(RuntimeError, match="name exceeds"):
+ custom_grader_runner._extract_custom_metrics(
+ {"custom_metrics": {"x" * (MAX_CUSTOM_METRIC_NAME_BYTES + 1): 0.5}}
+ )
+ with pytest.raises(RuntimeError, match="count exceeds"):
+ custom_grader_runner._extract_custom_metrics(
+ {"custom_metrics": {f"metric_{index:03d}": 0.5 for index in range(MAX_CUSTOM_METRICS + 1)}}
+ )
+
+
+@pytest.mark.parametrize(
+ "name",
+ [
+ "quality",
+ "secret_handling",
+ "token_efficiency",
+ "token_count",
+ "tokens",
+ "total_tokens",
+ "prompt_tokens",
+ "completion_tokens",
+ "max_tokens",
+ "last_token_usage",
+ "api_key_quality",
+ "passwords_quality",
+ "tokens_quality",
+ "privatekey_quality",
+ "sessiontoken_quality",
+ "sk-abcdefghijk",
+ "quality_sk-abcdefghijk",
+ "quality_nvapi-abcdefghijk",
+ "quality_crsr_0123456789abcdef",
+ "quality_SK-ABCDEFGHIJK",
+ "quality_NVAPI-ABCDEFGHIJK",
+ "quality_CRSR_0123456789ABCDEF",
+ "quality_ghp_" + ("a" * 36),
+ "quality_gho_" + ("a" * 36),
+ "quality_ghu_" + ("a" * 36),
+ "quality_ghs_" + ("a" * 36),
+ "quality_ghs_" + ("a" * 18) + ".-_" + ("b" * 18),
+ "quality_ghr_" + ("a" * 36),
+ "qualityghp_" + ("a" * 36),
+ "qualitygho_" + ("a" * 36) + "suffix",
+ "quality_github_pat_" + ("a" * 30),
+ "quality_" + "".join(("xoxb-", "1234567890-abcdefghijklmnopqrstuvwx")), # noqa: FLY002
+ "quality_" + "AIza" + ("A" * 35),
+ "quality_AIzA" + ("A" * 35),
+ "quality_glpat-" + ("a" * 20),
+ "https://user:pass@example.com",
+ " x",
+ "x" * (MAX_CUSTOM_METRIC_NAME_BYTES + 1),
+ ],
+)
+def test_custom_grader_runner_metric_name_policy_matches_collector(name: str) -> None:
+ assert custom_grader_runner._metric_name_is_publishable(name) is custom_metric_name_is_publishable(name)
+
+
+def test_custom_grader_runner_sanitizer_removes_unsafe_dict_valued_implicit_metric() -> None:
+ credential_metrics = [
+ "api_key_quality",
+ "ghp_" + ("a" * 36),
+ "gho_" + ("a" * 36),
+ "ghu_" + ("a" * 36),
+ "ghs_" + ("a" * 36),
+ "ghs_" + ("a" * 18) + ".-_" + ("b" * 18),
+ "ghr_" + ("a" * 36),
+ "qualityghp_" + ("a" * 36),
+ "qualitygho_" + ("a" * 36) + "suffix",
+ "github_pat_" + ("a" * 30),
+ "".join(("xoxb-", "1234567890-abcdefghijklmnopqrstuvwx")), # noqa: FLY002
+ "AIza" + ("A" * 35),
+ "glpat-" + ("a" * 20),
+ ]
+ unsafe_metrics = {name: {"score": 0.6, "reason": "must not survive"} for name in credential_metrics}
+ reward = {
+ "overall": 0.75,
+ **unsafe_metrics,
+ "quality": {"score": 0.8, "reason": "bounded evidence"},
+ }
+ custom_metrics = custom_grader_runner._extract_custom_metrics(reward)
+
+ assert custom_metrics == {"quality": 0.8}
+ assert custom_grader_runner._sanitized_custom_reward(reward, custom_metrics) == {
+ "overall": 0.75,
+ "quality": {"score": 0.8, "reason": "bounded evidence"},
+ }
+
+
+def test_custom_grader_runner_sanitizer_limits_both_detail_containers_to_validated_metrics() -> None:
+ reward = {
+ "overall": 0.75,
+ "custom_metrics": {"quality": 0.8, "api_key_quality": 0.6},
+ "details": {
+ "quality": {"reason": "bounded evidence"},
+ "api_key_quality": {"reason": "must not survive"},
+ },
+ "custom_details": {
+ "quality": {"report": "bounded evidence"},
+ "api_key_quality": {"report": "must not survive"},
+ },
+ }
+ custom_metrics = custom_grader_runner._extract_custom_metrics(reward)
+
+ assert custom_metrics == {"quality": 0.8}
+ assert custom_grader_runner._sanitized_custom_reward(reward, custom_metrics) == {
+ "overall": 0.75,
+ "custom_metrics": {"quality": 0.8},
+ "details": {"quality": {"reason": "bounded evidence"}},
+ "custom_details": {"quality": {"report": "bounded evidence"}},
+ }
+
+
+@pytest.mark.parametrize(
+ "rejected_value",
+ [{"reason": "nonnumeric"}, "nonnumeric", None],
+ ids=("dict", "string", "null"),
+)
+def test_custom_grader_runner_sanitizer_removes_nonnumeric_credential_shaped_keys(
+ rejected_value: object,
+) -> None:
+ rejected_name = "quality_sk-abcdefghijk"
+ reward = {
+ "overall": 0.75,
+ "custom_metrics": {"quality": 0.8},
+ rejected_name: rejected_value,
+ }
+ custom_metrics = custom_grader_runner._extract_custom_metrics(reward)
+
+ assert custom_metrics == {"quality": 0.8}
+ assert rejected_name not in custom_grader_runner._sanitized_custom_reward(reward, custom_metrics)
+
+
+@pytest.mark.parametrize(
+ "invalid_score",
+ ["nan", "inf", "-inf", "-0.01", "1.01", "1e308", str(10**400)],
+)
+def test_custom_grader_runner_rejects_invalid_text_overall_scores(invalid_score: str) -> None:
+ assert custom_grader_runner._score_from_text(invalid_score) is None
+
+
def _run_generated_test_sh(task_dir: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["bash", str(task_dir / "tests" / "test.sh")],
@@ -309,9 +500,11 @@ def test_generated_standard_grading_scripts_stop_after_evaluator_failure(
task_dir = tmp_path / grading_mode
_write_test_sh(task_dir, grading_mode=grading_mode, custom_grader=grading_mode == "default_plus_custom")
tests_dir = task_dir / "tests"
- (tests_dir / "eval.py").write_text("raise SystemExit(7)\n", encoding="utf-8")
+ evaluator_dir = tests_dir / _EVALUATOR_TESTS_SUBDIR
+ evaluator_dir.mkdir()
+ (evaluator_dir / "eval.py").write_text("raise SystemExit(7)\n", encoding="utf-8")
marker = task_dir / "custom-ran"
- (tests_dir / "custom_grader_runner.py").write_text(
+ (evaluator_dir / "custom_grader_runner.py").write_text(
"from pathlib import Path\nPath(" + repr(str(marker)) + ").write_text('ran')\n",
encoding="utf-8",
)
@@ -326,13 +519,17 @@ def test_generated_custom_only_script_accepts_overall_only_custom_reward(tmp_pat
task_dir = tmp_path / "custom-only"
_write_test_sh(task_dir, grading_mode="custom_only", custom_grader=True)
tests_dir = task_dir / "tests"
- shutil.copy2(_CUSTOM_RUNNER_TEMPLATE, tests_dir / "custom_grader_runner.py")
+ evaluator_dir = tests_dir / _EVALUATOR_TESTS_SUBDIR
+ evaluator_dir.mkdir()
+ shutil.copy2(_CUSTOM_RUNNER_TEMPLATE, evaluator_dir / "custom_grader_runner.py")
marker = task_dir / "custom-ran"
+ (tests_dir / "custom_helper.py").write_text("OVERALL = 0.75\n", encoding="utf-8")
(tests_dir / "grader.py").write_text(
"import json, os\n"
"from pathlib import Path\n"
+ "from custom_helper import OVERALL\n"
f"Path({str(marker)!r}).write_text('ran')\n"
- "Path(os.environ['HARBOR_REWARD_JSON']).write_text(json.dumps({'overall': 0.75}))\n",
+ "Path(os.environ['HARBOR_REWARD_JSON']).write_text(json.dumps({'overall': OVERALL}))\n",
encoding="utf-8",
)
verifier_dir = task_dir / "verifier"
@@ -357,3 +554,301 @@ def test_generated_custom_only_script_accepts_overall_only_custom_reward(tmp_pat
assert marker.read_text(encoding="utf-8") == "ran"
assert json.loads(reward_json.read_text(encoding="utf-8")) == {"overall": 0.75}
assert reward_txt.read_text(encoding="utf-8") == "0.75"
+
+
+@pytest.mark.parametrize("mode", ["default_plus_custom", "custom_only"])
+def test_generated_custom_grader_rejects_metric_overflow_before_retaining_raw_names(
+ tmp_path: Path,
+ mode: str,
+) -> None:
+ tests_dir = tmp_path / "tests"
+ verifier_dir = tmp_path / "verifier"
+ tests_dir.mkdir()
+ verifier_dir.mkdir()
+ raw_names = [f"metric_{index:03d}" for index in range(MAX_CUSTOM_METRICS + 1)]
+ (tests_dir / "grader.py").write_text(
+ "import json, os\n"
+ "from pathlib import Path\n"
+ f"payload = {{'overall': 0.75, 'custom_metrics': {dict.fromkeys(raw_names, 0.5)!r}}}\n"
+ "Path(os.environ['HARBOR_REWARD_JSON']).write_text(json.dumps(payload))\n",
+ encoding="utf-8",
+ )
+ reward_json = verifier_dir / "reward.json"
+ reward_txt = verifier_dir / "reward.txt"
+ skill_evaluator_reward = verifier_dir / "skill_evaluator_reward.json"
+ custom_reward = verifier_dir / "custom_reward.json"
+ if mode == "default_plus_custom":
+ skill_evaluator_reward.write_text(
+ json.dumps(
+ {
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(RESERVED_METRIC_NAMES & set(custom_grader_runner.DEFAULT_METRICS), 1.0),
+ "overall": 1.0,
+ }
+ ),
+ encoding="utf-8",
+ )
+ reward_txt.write_text("1.0", encoding="utf-8")
+
+ completed = subprocess.run(
+ [sys.executable, str(_CUSTOM_RUNNER_TEMPLATE), "--mode", mode],
+ check=False,
+ capture_output=True,
+ text=True,
+ env={
+ **os.environ,
+ "HARBOR_TESTS_DIR": str(tests_dir),
+ "HARBOR_VERIFIER_DIR": str(verifier_dir),
+ "HARBOR_REWARD_JSON": str(reward_json),
+ "HARBOR_REWARD_TXT": str(reward_txt),
+ "HARBOR_SKILL_EVALUATOR_REWARD_JSON": str(skill_evaluator_reward),
+ "HARBOR_CUSTOM_REWARD_JSON": str(custom_reward),
+ "HARBOR_GRADER": str(tests_dir / "grader.py"),
+ "HARBOR_GRADER_SH": str(tests_dir / "grader.sh"),
+ },
+ )
+
+ assert completed.returncode != 0
+ retained_text = custom_reward.read_text(encoding="utf-8")
+ retained = json.loads(retained_text)
+ assert retained["overall"] == 0.0
+ assert "count exceeds" in retained["error"]
+ assert all(name not in retained_text for name in raw_names)
+ assert json.loads(reward_json.read_text(encoding="utf-8")) == {"overall": 0.0}
+
+
+@pytest.mark.parametrize("mode", ["default_plus_custom", "custom_only"])
+def test_generated_custom_grader_omits_unsafe_dict_valued_metric_from_every_artifact(
+ tmp_path: Path,
+ mode: str,
+) -> None:
+ tests_dir = tmp_path / "tests"
+ verifier_dir = tmp_path / "verifier"
+ tests_dir.mkdir()
+ verifier_dir.mkdir()
+ credential_metrics = [
+ "api_key_quality",
+ "ghp_" + ("a" * 36),
+ "gho_" + ("a" * 36),
+ "ghu_" + ("a" * 36),
+ "ghs_" + ("a" * 36),
+ "ghs_" + ("a" * 18) + ".-_" + ("b" * 18),
+ "ghr_" + ("a" * 36),
+ "qualityghp_" + ("a" * 36),
+ "qualitygho_" + ("a" * 36) + "suffix",
+ "github_pat_" + ("a" * 30),
+ "".join(("xoxb-", "1234567890-abcdefghijklmnopqrstuvwx")), # noqa: FLY002
+ "AIza" + ("A" * 35),
+ "glpat-" + ("a" * 20),
+ ]
+ unsafe_metrics = {name: {"score": 0.6, "reason": "must not survive"} for name in credential_metrics}
+ detail_only_name = "detail_only_not_a_metric"
+ custom_details = {
+ "quality": {"report": "bounded evidence"},
+ credential_metrics[0]: {"report": "must not survive"},
+ detail_only_name: {"report": "must not survive"},
+ }
+ (tests_dir / "grader.py").write_text(
+ "import json, os\n"
+ "from pathlib import Path\n"
+ "payload = {'overall': 0.75, "
+ f"**{unsafe_metrics!r}, "
+ "'quality': {'score': 0.8, 'reason': 'bounded evidence'}, "
+ f"'custom_details': {custom_details!r}}}\n"
+ "Path(os.environ['HARBOR_REWARD_JSON']).write_text(json.dumps(payload))\n",
+ encoding="utf-8",
+ )
+ reward_json = verifier_dir / "reward.json"
+ reward_txt = verifier_dir / "reward.txt"
+ skill_evaluator_reward = verifier_dir / "skill_evaluator_reward.json"
+ custom_reward = verifier_dir / "custom_reward.json"
+ if mode == "default_plus_custom":
+ skill_evaluator_reward.write_text(
+ json.dumps(
+ {
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(custom_grader_runner.DEFAULT_METRICS, 1.0),
+ "overall": 1.0,
+ }
+ ),
+ encoding="utf-8",
+ )
+ reward_txt.write_text("1.0", encoding="utf-8")
+
+ completed = subprocess.run(
+ [sys.executable, str(_CUSTOM_RUNNER_TEMPLATE), "--mode", mode],
+ check=False,
+ capture_output=True,
+ text=True,
+ env={
+ **os.environ,
+ "HARBOR_TESTS_DIR": str(tests_dir),
+ "HARBOR_VERIFIER_DIR": str(verifier_dir),
+ "HARBOR_REWARD_JSON": str(reward_json),
+ "HARBOR_REWARD_TXT": str(reward_txt),
+ "HARBOR_SKILL_EVALUATOR_REWARD_JSON": str(skill_evaluator_reward),
+ "HARBOR_CUSTOM_REWARD_JSON": str(custom_reward),
+ "HARBOR_GRADER": str(tests_dir / "grader.py"),
+ "HARBOR_GRADER_SH": str(tests_dir / "grader.sh"),
+ },
+ )
+
+ assert completed.returncode == 0, completed.stderr
+ for artifact in (reward_json, custom_reward, skill_evaluator_reward):
+ if not artifact.exists():
+ continue
+ text = artifact.read_text(encoding="utf-8")
+ assert all(name not in text for name in credential_metrics)
+ assert detail_only_name not in text
+ assert "must not survive" not in text
+ assert json.loads(reward_json.read_text(encoding="utf-8"))["quality"] == 0.8
+ custom_payload = json.loads(custom_reward.read_text(encoding="utf-8"))
+ assert custom_payload["quality"]["score"] == 0.8
+ assert custom_payload["custom_details"] == {"quality": {"report": "bounded evidence"}}
+
+
+@pytest.mark.parametrize("mode", ["default_plus_custom", "custom_only"])
+def test_generated_custom_grader_mixed_metric_surfaces_match_collector_without_credential_drift(
+ tmp_path: Path,
+ mode: str,
+) -> None:
+ tests_dir = tmp_path / "tests"
+ verifier_dir = tmp_path / "verifier"
+ tests_dir.mkdir()
+ verifier_dir.mkdir()
+ credential_metrics = [
+ "api_key_quality",
+ "ghp_" + ("a" * 36),
+ "gho_" + ("a" * 36),
+ "ghu_" + ("a" * 36),
+ "ghs_" + ("a" * 36),
+ "ghs_" + ("a" * 18) + ".-_" + ("b" * 18),
+ "ghr_" + ("a" * 36),
+ "qualityghp_" + ("a" * 36),
+ "qualitygho_" + ("a" * 36) + "suffix",
+ "github_pat_" + ("a" * 30),
+ "".join(("xoxb-", "1234567890-abcdefghijklmnopqrstuvwx")), # noqa: FLY002
+ "AIza" + ("A" * 35),
+ "glpat-" + ("a" * 20),
+ "quality_sk-abcdefghijk_nonnumeric_dict",
+ "quality_nvapi-abcdefghijk_nonnumeric_string",
+ "quality_crsr_0123456789abcdef_nonnumeric_null",
+ ]
+ unsafe_metrics = {name: {"score": 0.6, "reason": "must not survive"} for name in credential_metrics}
+ unsafe_metrics.update(
+ {
+ "quality_sk-abcdefghijk_nonnumeric_dict": {"reason": "must not survive"},
+ "quality_nvapi-abcdefghijk_nonnumeric_string": "must not survive",
+ "quality_crsr_0123456789abcdef_nonnumeric_null": None,
+ }
+ )
+ (tests_dir / "grader.py").write_text(
+ "import json, os\n"
+ "from pathlib import Path\n"
+ "payload = {'overall': 0.75, 'custom_metrics': {'quality': 0.8}, "
+ "'domain_score': {'score': 0.7, 'reason': 'bounded evidence'}, "
+ f"**{unsafe_metrics!r}}}\n"
+ "Path(os.environ['HARBOR_REWARD_JSON']).write_text(json.dumps(payload))\n",
+ encoding="utf-8",
+ )
+ reward_json = verifier_dir / "reward.json"
+ reward_txt = verifier_dir / "reward.txt"
+ skill_evaluator_reward = verifier_dir / "skill_evaluator_reward.json"
+ custom_reward = verifier_dir / "custom_reward.json"
+ if mode == "default_plus_custom":
+ skill_evaluator_reward.write_text(
+ json.dumps(
+ {
+ "metric_set": DEFAULT_METRIC_SET,
+ **dict.fromkeys(custom_grader_runner.DEFAULT_METRICS, 1.0),
+ "overall": 1.0,
+ }
+ ),
+ encoding="utf-8",
+ )
+ reward_txt.write_text("1.0", encoding="utf-8")
+
+ completed = subprocess.run(
+ [sys.executable, str(_CUSTOM_RUNNER_TEMPLATE), "--mode", mode],
+ check=False,
+ capture_output=True,
+ text=True,
+ env={
+ **os.environ,
+ "HARBOR_TESTS_DIR": str(tests_dir),
+ "HARBOR_VERIFIER_DIR": str(verifier_dir),
+ "HARBOR_REWARD_JSON": str(reward_json),
+ "HARBOR_REWARD_TXT": str(reward_txt),
+ "HARBOR_SKILL_EVALUATOR_REWARD_JSON": str(skill_evaluator_reward),
+ "HARBOR_CUSTOM_REWARD_JSON": str(custom_reward),
+ "HARBOR_GRADER": str(tests_dir / "grader.py"),
+ "HARBOR_GRADER_SH": str(tests_dir / "grader.sh"),
+ },
+ )
+
+ assert completed.returncode == 0, completed.stderr
+ custom_payload = json.loads(custom_reward.read_text(encoding="utf-8"))
+ harbor_payload = json.loads(reward_json.read_text(encoding="utf-8"))
+ expected = {"domain_score": 0.7, "quality": 0.8}
+ assert extract_custom_metrics(custom_payload) == expected
+ assert extract_custom_metrics(harbor_payload) == expected
+ for artifact in (reward_json, custom_reward, skill_evaluator_reward):
+ if artifact.exists():
+ text = artifact.read_text(encoding="utf-8")
+ assert all(name not in text for name in credential_metrics)
+
+
+@pytest.mark.parametrize("source", ["reward_json", "reward_txt"])
+@pytest.mark.parametrize("invalid_score", ["nan", "inf", "-inf", "-0.01", "1.01", "1e308"])
+def test_generated_custom_only_script_rejects_invalid_overall_score(
+ tmp_path: Path,
+ source: str,
+ invalid_score: str,
+) -> None:
+ task_dir = tmp_path / f"custom-only-{source}"
+ _write_test_sh(task_dir, grading_mode="custom_only", custom_grader=True)
+ tests_dir = task_dir / "tests"
+ evaluator_dir = tests_dir / _EVALUATOR_TESTS_SUBDIR
+ evaluator_dir.mkdir()
+ shutil.copy2(_CUSTOM_RUNNER_TEMPLATE, evaluator_dir / "custom_grader_runner.py")
+ grader_lines = [
+ "import json, os",
+ "from pathlib import Path",
+ "reward_json = Path(os.environ['HARBOR_REWARD_JSON'])",
+ "reward_txt = Path(os.environ['HARBOR_REWARD_TXT'])",
+ ]
+ if source == "reward_json":
+ grader_lines.append(f"reward_json.write_text(json.dumps({{'overall': float({invalid_score!r})}}))")
+ else:
+ grader_lines.extend(
+ [
+ "reward_json.write_text('{}')",
+ f"reward_txt.write_text({invalid_score!r})",
+ ]
+ )
+ (tests_dir / "grader.py").write_text("\n".join(grader_lines) + "\n", encoding="utf-8")
+ verifier_dir = task_dir / "verifier"
+ verifier_dir.mkdir()
+ reward_json = verifier_dir / "reward.json"
+ reward_txt = verifier_dir / "reward.txt"
+ custom_reward_json = verifier_dir / "custom_reward.json"
+
+ completed = _run_generated_test_sh(
+ task_dir,
+ {
+ "HARBOR_TESTS_DIR": str(tests_dir),
+ "HARBOR_VERIFIER_DIR": str(verifier_dir),
+ "HARBOR_REWARD_JSON": str(reward_json),
+ "HARBOR_REWARD_TXT": str(reward_txt),
+ "HARBOR_CUSTOM_REWARD_JSON": str(custom_reward_json),
+ "HARBOR_GRADER": str(tests_dir / "grader.py"),
+ "HARBOR_GRADER_SH": str(tests_dir / "grader.sh"),
+ },
+ )
+
+ assert completed.returncode != 0
+ failure = json.loads(custom_reward_json.read_text(encoding="utf-8"))
+ assert failure["overall"] == 0.0
+ assert "between 0.0 and 1.0" in failure["error"]
+ assert json.loads(reward_json.read_text(encoding="utf-8")) == {"overall": 0.0}
+ assert reward_txt.read_text(encoding="utf-8") == "0.0"
diff --git a/tests/tier3/test_suggestion_grounding.py b/tests/tier3/test_suggestion_grounding.py
index 392158a7..ac57914a 100644
--- a/tests/tier3/test_suggestion_grounding.py
+++ b/tests/tier3/test_suggestion_grounding.py
@@ -1,6 +1,10 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
+import json
+
+import pytest
+
from skillevaluator.tier3.harbor import report
@@ -77,6 +81,84 @@ def test_findings_carry_evidence_refs():
assert goal["evidence_refs"][0]["json_pointer"] == "/steps/14"
+def test_findings_normalize_legacy_string_evidence_refs_without_crashing():
+ reward = _reward(0.1)
+ reward["details"]["goal_accuracy"]["evidence_refs"] = ["trajectory.json#/steps/14"]
+
+ findings = report._extract_findings([reward])
+
+ goal = next(finding for finding in findings if finding["metric"] == "goal_accuracy")
+ assert goal["evidence_refs"] == [
+ {
+ "source": "trajectory.json",
+ "json_pointer": "/steps/14",
+ "kind": "evidence",
+ }
+ ]
+
+
+@pytest.mark.parametrize(
+ ("metric", "malformed_detail"),
+ [
+ ("behavior_check", {"results": ["malformed"]}),
+ ("accuracy", []),
+ ("accuracy", {"criteria": [], "reason": 42}),
+ ("goal_accuracy", {"findings": "malformed", "reason": {"nested": "value"}}),
+ ],
+)
+@pytest.mark.parametrize("metric_score", [0.1, 0.9], ids=["failing", "passing"])
+def test_findings_tolerate_malformed_nested_detail_shapes(metric, malformed_detail, metric_score):
+ reward = _reward(metric_score)
+ reward[metric] = metric_score
+ reward["details"][metric] = malformed_detail
+
+ findings = report._extract_findings([reward])
+
+ finding = next(item for item in findings if item["metric"] == metric)
+ assert finding["score"] == pytest.approx(metric_score)
+ assert all(isinstance(reason, str) and len(reason) <= 512 for reason in finding["reasons"])
+
+
+def test_findings_report_rejects_overflowing_reward_numbers_without_crashing():
+ reward = _reward(10**400)
+ reward["details"]["goal_accuracy"]["score"] = 10**400
+
+ assert report._extract_findings([reward]) == []
+ assert report._prioritized_evidence_rewards([reward]) == [reward]
+
+
+@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), float("-inf"), 10**400])
+def test_findings_best_agent_ignores_invalid_legacy_summary_numbers(invalid: float | int):
+ invalid_scores = dict.fromkeys(report.DISPLAY_METRICS, 1.0)
+ invalid_scores[report.DISPLAY_METRICS[0]] = invalid
+ agents = {
+ "invalid": {
+ "execution_status": "succeeded",
+ "with_skill": invalid_scores,
+ },
+ "valid": {
+ "execution_status": "succeeded",
+ "with_skill": dict.fromkeys(report.DISPLAY_METRICS, 0.5),
+ },
+ }
+
+ assert report._pick_best_agent(agents) == "valid"
+
+
+def test_passing_suggestions_count_mixed_current_and_legacy_logical_trials():
+ suggestions = report._passing_skill_suggestions(
+ [],
+ [
+ {"trial_id": "current-attempt"},
+ {"trial_id": "current-attempt"},
+ {"entry_id": "legacy-one"},
+ {"entry_id": "legacy-two"},
+ ],
+ )
+
+ assert any("currently 3" in suggestion for suggestion in suggestions)
+
+
def test_generate_suggestions_prompt_includes_refs_and_uses_larger_budget(monkeypatch):
captured = {}
@@ -134,6 +216,38 @@ def test_findings_artifact_includes_suggestions_v2(tmp_path):
assert "suggestions_v2" in payload and payload["suggestions_v2"][0]["dimension"] == "goal_accuracy"
+def test_findings_artifact_bounds_evidence_across_multiple_large_rewards(tmp_path):
+ rewards = []
+ for index in range(8):
+ reward = _reward(0.1)
+ reward["entry_id"] = f"case-{index}"
+ reward["details"]["goal_accuracy"]["evidence_refs"] = [
+ {
+ "source": f"trajectory-{index}.json",
+ "json_pointer": f"/steps/{index}",
+ "kind": "tool_call",
+ "excerpt": "x" * 1_000_000,
+ }
+ ]
+ rewards.append(reward)
+
+ findings = report._extract_findings(rewards)
+ art = report._write_findings_artifact(
+ results_dir=tmp_path,
+ skill_name="demo",
+ agent="codex",
+ findings=findings,
+ suggestions=[],
+ suggestion_mode="not_generated",
+ )
+ payload = json.loads(art.read_text(encoding="utf-8"))
+ goal = next(finding for finding in payload["findings"] if finding["metric"] == "goal_accuracy")
+
+ assert art.stat().st_size <= report.report_data._MAX_JSON_BYTES
+ assert len(goal["evidence_refs"]) == report._MAX_FINDING_EVIDENCE_REFS
+ assert all(len(ref["excerpt"]) < 1_000 for ref in goal["evidence_refs"])
+
+
# --- New tests for dict-shaped evidence_refs in suggestions_v2 ---
diff --git a/uv.lock b/uv.lock
index f6fb5c20..8a89e0b8 100644
--- a/uv.lock
+++ b/uv.lock
@@ -19,7 +19,7 @@ wheels = [
[[package]]
name = "aiohttp"
-version = "3.14.1"
+version = "3.14.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohappyeyeballs" },
@@ -31,49 +31,49 @@ dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
{ name = "yarl" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" },
- { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" },
- { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" },
- { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" },
- { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" },
- { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" },
- { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" },
- { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" },
- { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" },
- { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" },
- { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" },
- { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" },
- { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" },
- { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" },
- { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" },
- { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" },
- { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" },
- { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" },
- { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" },
- { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" },
- { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" },
- { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" },
- { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" },
- { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" },
- { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" },
- { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" },
- { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" },
- { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" },
- { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" },
- { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" },
- { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" },
- { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" },
- { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" },
- { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" },
- { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" },
- { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" },
- { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" },
- { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" },
- { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" },
- { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" },
- { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" },
+ { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" },
+ { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" },
+ { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" },
+ { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" },
+ { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" },
+ { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" },
+ { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" },
+ { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" },
+ { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" },
+ { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" },
+ { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" },
+ { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" },
+ { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" },
+ { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" },
+ { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" },
+ { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" },
+ { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" },
+ { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" },
+ { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" },
+ { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" },
+ { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" },
]
[[package]]
@@ -317,24 +317,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" },
]
-[[package]]
-name = "claude-agent-sdk"
-version = "0.2.110"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "anyio" },
- { name = "mcp" },
- { name = "sniffio" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/bb/98/8fdab35ed9e1a36bc7afab4d390cc5002094a4950996c079da9aa4541cc4/claude_agent_sdk-0.2.110.tar.gz", hash = "sha256:538b548bac07a22f65686abab063a902ac76ba35989d0f073c942f96248e9fa3", size = 255632, upload-time = "2026-06-24T22:11:52.342Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/aa/93/29d4fdaa13e69034faf8d3503df915b07c820e2c08e3d6a7515149cde5bb/claude_agent_sdk-0.2.110-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fed0e0f4804d9f9cff80ab7d1b44142ebd1046cdd29ca74caef4c92c35fff8d8", size = 64924533, upload-time = "2026-06-24T22:11:55.612Z" },
- { url = "https://files.pythonhosted.org/packages/aa/03/b40bb673cd93cdc3928262c1be75fde34a7bed4bf2c2c20e04218e2005ea/claude_agent_sdk-0.2.110-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:62b23869d46cef6f6ff1d00ceaa5e846e2f1d297478421c835efb8fe99369d4f", size = 69704449, upload-time = "2026-06-24T22:11:59.149Z" },
- { url = "https://files.pythonhosted.org/packages/f9/18/ab67cb5ce641333385bed55ed8e9665c00f7d30d1f6ab12f8463ddb7695f/claude_agent_sdk-0.2.110-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:324e49553c6303d6b267217dc2912652b97af2bc96503efd12095ae915b46b83", size = 74879555, upload-time = "2026-06-24T22:12:03.25Z" },
- { url = "https://files.pythonhosted.org/packages/91/88/3627d7d14310cfec66977551263e219365244a906fc7ca1209fb0c3a6cec/claude_agent_sdk-0.2.110-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:56371dd7a2c66c0bd497dc0b3cab4193a228b196f600676393d69c0ecee37cfb", size = 75924237, upload-time = "2026-06-24T22:12:07.183Z" },
- { url = "https://files.pythonhosted.org/packages/49/79/c9066c5c387d42c19a4b675ec1ff5219f8920cfda8ff8b527119fd69b774/claude_agent_sdk-0.2.110-py3-none-win_amd64.whl", hash = "sha256:4235d4de6d685a189c12612095ab192b759280ede1f3aed0c3e784d52c3555f9", size = 75448209, upload-time = "2026-06-24T22:12:11.283Z" },
-]
-
[[package]]
name = "click"
version = "8.4.2"
@@ -397,39 +379,39 @@ wheels = [
[[package]]
name = "cryptography"
-version = "49.0.0"
+version = "50.0.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cffi", marker = "platform_python_implementation != 'PyPy'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/1f/99/d1c90d6041656cc6ee229dc99cd67fd0cd5aec3c5f7d72fffc27cc750054/cryptography-49.0.0.tar.gz", hash = "sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493", size = 854345, upload-time = "2026-06-12T20:02:30.512Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/9b/22/adf66990e63584a68dfb50c24f48a125c07b1699899381c8151e63ed458c/cryptography-49.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db", size = 4032100, upload-time = "2026-06-12T20:02:32.143Z" },
- { url = "https://files.pythonhosted.org/packages/09/41/3797cfaf69cae04a13ee78ebd83f0678d9c02b4779d21ce24445326f1a69/cryptography-49.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db", size = 4692978, upload-time = "2026-06-12T20:01:21.305Z" },
- { url = "https://files.pythonhosted.org/packages/e6/8b/43011f7ebe515a8aa20d61f290a326cd890c2e738e16e59eaff8d9c3a412/cryptography-49.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325", size = 4716422, upload-time = "2026-06-12T20:01:48.566Z" },
- { url = "https://files.pythonhosted.org/packages/4a/91/01ce7303a4579e6d3a6abef01bd322848e9ea7a219adcabc5048b9033571/cryptography-49.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2", size = 4700503, upload-time = "2026-06-12T20:02:47.091Z" },
- { url = "https://files.pythonhosted.org/packages/62/99/a2c95cf8293f07491e9e27c20cc4dcd18176d944e674679adeb1d0173fd6/cryptography-49.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b", size = 5309779, upload-time = "2026-06-12T20:02:08.987Z" },
- { url = "https://files.pythonhosted.org/packages/20/2c/0622f20ff02b2ef32558733443805dc82fd4c275be01b2d19d14676f3a1b/cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6", size = 4749683, upload-time = "2026-06-12T20:02:03.335Z" },
- { url = "https://files.pythonhosted.org/packages/a3/5b/c5246635d5fd3b64e0d45ae10e99fd32fe9676a79915ccfe5a61ba9af1a5/cryptography-49.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c", size = 4337874, upload-time = "2026-06-12T20:02:54.323Z" },
- { url = "https://files.pythonhosted.org/packages/6d/88/05563c7fe2e914e87d1a536d06fe83e66b4e1d95cb593e05aea375531da8/cryptography-49.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7", size = 4700283, upload-time = "2026-06-12T20:01:34.822Z" },
- { url = "https://files.pythonhosted.org/packages/c4/b6/d7696e4e890d6ae1469935164c9e5215c557671cb78d6e3f458ccceaa632/cryptography-49.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68", size = 5265844, upload-time = "2026-06-12T20:01:24.09Z" },
- { url = "https://files.pythonhosted.org/packages/a9/3c/f3ad17eecc1a57b0ba236dc01f90e783c51f4a2f35f64777cc4f47a184b2/cryptography-49.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9", size = 4749290, upload-time = "2026-06-12T20:01:30.848Z" },
- { url = "https://files.pythonhosted.org/packages/4f/01/339573cf1023163a400b0b5d16f6d507de413b9f60be6fd1b77feeaf6737/cryptography-49.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f", size = 4834612, upload-time = "2026-06-12T20:01:29.246Z" },
- { url = "https://files.pythonhosted.org/packages/71/fd/577302e213a1be9468f92d1afef66fcf1ef83d516819d9992ca547f592bd/cryptography-49.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459", size = 4980804, upload-time = "2026-06-12T20:01:42.853Z" },
- { url = "https://files.pythonhosted.org/packages/1f/09/f42b1d190c5ba75f72062a387f8030d1d75f6ab035788f1d9c4b01de6525/cryptography-49.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e", size = 3810026, upload-time = "2026-06-12T20:02:39.262Z" },
- { url = "https://files.pythonhosted.org/packages/19/2a/5bb823f5bedcf80718cea7fbc95ec5515cca3769633c4b01a32be7f30e7c/cryptography-49.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a", size = 4025947, upload-time = "2026-06-12T20:01:25.745Z" },
- { url = "https://files.pythonhosted.org/packages/3d/df/40577043ca124e17012f408ddddaeb213b856336ac82ddb3bc915f39e29f/cryptography-49.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4", size = 4692429, upload-time = "2026-06-12T20:01:53.628Z" },
- { url = "https://files.pythonhosted.org/packages/2c/99/2d13299eb3dd27b02dcfaafcc91d6b5cb3329f7cbd6d8f51921acd566c1a/cryptography-49.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18", size = 4700968, upload-time = "2026-06-12T20:02:45.383Z" },
- { url = "https://files.pythonhosted.org/packages/a5/4d/9c0cd02f95e2602dd5e563da149ee0830abef3537be8b34dc56281ebe27a/cryptography-49.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69", size = 4697758, upload-time = "2026-06-12T20:01:41.13Z" },
- { url = "https://files.pythonhosted.org/packages/24/01/186c825898477d77e2324d5360fefe622ff1d8d1963ec0554e2cada8ec77/cryptography-49.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64", size = 5298863, upload-time = "2026-06-12T20:02:24.579Z" },
- { url = "https://files.pythonhosted.org/packages/b8/7b/62cbbab75d0659865bf0273790031544a0b16c8072d258f9428dcd8190dc/cryptography-49.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21", size = 4735983, upload-time = "2026-06-12T20:01:50.14Z" },
- { url = "https://files.pythonhosted.org/packages/6c/72/3e798c064bc39e471008075d0f9bc9daf77a80879c092e4a8e170c585ed4/cryptography-49.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9", size = 4334173, upload-time = "2026-06-12T20:01:44.743Z" },
- { url = "https://files.pythonhosted.org/packages/f0/ee/6fca21d1ac73e06f8bef71940abfd4d2f6472b4bca284d770f32bd4086f6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc", size = 4697298, upload-time = "2026-06-12T20:02:20.918Z" },
- { url = "https://files.pythonhosted.org/packages/67/d0/a5fcd3515f0bae49a7b6d0413cc1bdccdcc1fc0047037a0d480642cdc5d6/cryptography-49.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8", size = 5254338, upload-time = "2026-06-12T20:02:22.737Z" },
- { url = "https://files.pythonhosted.org/packages/a0/84/84fe36f19caf857d61cb7fc9c63035a47ffabd84ea12d1d393148efa3615/cryptography-49.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36", size = 4735650, upload-time = "2026-06-12T20:02:41.389Z" },
- { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" },
- { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" },
- { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" },
+sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" },
+ { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" },
+ { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" },
+ { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" },
+ { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" },
+ { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" },
+ { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" },
+ { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" },
+ { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" },
+ { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" },
+ { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" },
+ { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" },
+ { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" },
+ { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" },
+ { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" },
+ { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" },
+ { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" },
]
[[package]]
@@ -440,10 +422,10 @@ resolution-markers = [
"sys_platform == 'linux'",
]
dependencies = [
- { name = "license-expression", marker = "sys_platform == 'linux'" },
- { name = "packageurl-python", marker = "sys_platform == 'linux'" },
- { name = "py-serializable", marker = "sys_platform == 'linux'" },
- { name = "sortedcontainers", marker = "sys_platform == 'linux'" },
+ { name = "license-expression" },
+ { name = "packageurl-python" },
+ { name = "py-serializable" },
+ { name = "sortedcontainers" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/fc/abaad5482f7b59c9a0a9d8f354ce4ce23346d582a0d85730b559562bbeb4/cyclonedx_python_lib-9.1.0.tar.gz", hash = "sha256:86935f2c88a7b47a529b93c724dbd3e903bc573f6f8bd977628a7ca1b5dadea1", size = 1048735, upload-time = "2025-02-27T17:23:40.367Z" }
wheels = [
@@ -460,42 +442,17 @@ resolution-markers = [
"sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'",
]
dependencies = [
- { name = "license-expression", marker = "sys_platform != 'linux'" },
- { name = "packageurl-python", marker = "sys_platform != 'linux'" },
- { name = "py-serializable", marker = "sys_platform != 'linux'" },
- { name = "sortedcontainers", marker = "sys_platform != 'linux'" },
- { name = "typing-extensions", marker = "python_full_version < '3.13' and sys_platform != 'linux'" },
+ { name = "license-expression" },
+ { name = "packageurl-python" },
+ { name = "py-serializable" },
+ { name = "sortedcontainers" },
+ { name = "typing-extensions", marker = "python_full_version < '3.13'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/75/c9/5d0ccdd19bc7d8ab803b90695c1706aa2ea8529685d18e682dc2524d2630/cyclonedx_python_lib-11.11.0.tar.gz", hash = "sha256:4b3194db72b613717f2912447e67ab618c75ff7dcac6c4af3c0e9e1ac617c102", size = 1442983, upload-time = "2026-06-17T11:57:49.055Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/22/f3/56ccb2884aaa3db5622368e5191a3384b15f35392aa93df8b2f508c660d2/cyclonedx_python_lib-11.11.0-py3-none-any.whl", hash = "sha256:3049fc83e06a059b5c5907a527625a8ed5073caab10607ed4c9e5503b590fd44", size = 528689, upload-time = "2026-06-17T11:57:47.358Z" },
]
-[[package]]
-name = "datasets"
-version = "5.0.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "dill" },
- { name = "filelock" },
- { name = "fsspec", extra = ["http"] },
- { name = "httpx" },
- { name = "huggingface-hub" },
- { name = "multiprocess" },
- { name = "numpy" },
- { name = "packaging" },
- { name = "pandas" },
- { name = "pyarrow" },
- { name = "pyyaml" },
- { name = "requests" },
- { name = "tqdm" },
- { name = "xxhash" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/d9/85/ce4f780c32f7e36d71257f1c27e8ba898ebe379cb54f211f5f2013f2c219/datasets-5.0.0.tar.gz", hash = "sha256:83dbbbdb07a33b82192b8c419deb18739b138ee2ce1a322d55ce6b100954ec1a", size = 631708, upload-time = "2026-06-05T13:18:26.124Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/05/66/73034ad30b59f13439b75e620989dacba4c047256e358ba7c2e9ec98ea22/datasets-5.0.0-py3-none-any.whl", hash = "sha256:7dd34927a0fd7046e98aad5cb9430e699c373238a15befa7b9bf22b991a7fee6", size = 555084, upload-time = "2026-06-05T13:18:24.435Z" },
-]
-
[[package]]
name = "defusedxml"
version = "0.7.1"
@@ -517,15 +474,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" },
]
-[[package]]
-name = "dill"
-version = "0.4.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" },
-]
-
[[package]]
name = "dirhash"
version = "0.5.0"
@@ -686,11 +634,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" },
]
-[package.optional-dependencies]
-http = [
- { name = "aiohttp" },
-]
-
[[package]]
name = "h11"
version = "0.16.0"
@@ -702,37 +645,37 @@ wheels = [
[[package]]
name = "h2"
-version = "4.3.0"
+version = "4.4.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "hpack" },
{ name = "hyperframe" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e7/85/7c366e69d84c17bb778fe41419e1fbcce3033d5b7ce29bbffff0a98b859f/h2-4.4.1.tar.gz", hash = "sha256:4e866ffb1a869ae14dd9b5e6beb5c24a13da0495ad72b65925ded182521c1516", size = 2157281, upload-time = "2026-08-03T11:45:09.509Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/22/e85faf23bd72a92d1921e37d674ca56eb298a3c8be31fdecef0ff2b3aaac/h2-4.4.1-py3-none-any.whl", hash = "sha256:0e25f1462b23c9cb82d9eb02e28bc706dac2a68cb457c6a0d74d63c8a2a5d0e6", size = 62636, upload-time = "2026-08-03T11:44:59.164Z" },
]
[[package]]
name = "harbor"
-version = "0.13.2"
+version = "0.22.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "claude-agent-sdk" },
- { name = "datasets" },
{ name = "dirhash" },
{ name = "fastapi" },
+ { name = "filelock" },
{ name = "httpx" },
{ name = "jinja2" },
{ name = "litellm" },
{ name = "packaging" },
{ name = "pathspec" },
+ { name = "platformdirs" },
{ name = "pydantic" },
+ { name = "pyjwt" },
{ name = "python-dotenv" },
{ name = "pyyaml" },
{ name = "requests" },
{ name = "rich" },
- { name = "ruff" },
{ name = "shortuuid" },
{ name = "supabase" },
{ name = "tenacity" },
@@ -740,9 +683,9 @@ dependencies = [
{ name = "typer" },
{ name = "uvicorn" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/09/d2/8ea1fa703ebc8acd15fffc020555259dd1bb4f4188e830b5467ca0fd1950/harbor-0.13.2.tar.gz", hash = "sha256:e5cf94f9dac9bb465d61f37a689679b69ecd1987bae1ad48714dc641bea2f699", size = 1154370, upload-time = "2026-06-11T00:15:35.44Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/12/a0/a37532fb647b93ef709b63a7fbdf613a84837a569fdd88f69dd7105bf093/harbor-0.22.0.tar.gz", hash = "sha256:becf0ce354026cc37899855e0a0d2687cd5188034a43635849245069aad0938b", size = 1832554, upload-time = "2026-08-22T03:22:39.798Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/e6/ae/fc41d4d2bf830a878cf778877aafe90c05ef45651dab05dee3df7e463e1d/harbor-0.13.2-py3-none-any.whl", hash = "sha256:7a60207965755268fb0a4748a553ce92ed91ce7b32805e7f0c8fea1ed61567b6", size = 1313409, upload-time = "2026-06-11T00:15:36.98Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/12/d517f1ca18738f7be78a210b13264c64bace531fdf644632128966aca530/harbor-0.22.0-py3-none-any.whl", hash = "sha256:4c4c6571b3d160ed0cb45b82918136751fb08e7b8596412723ac00dde12eeabb", size = 2058718, upload-time = "2026-08-22T03:22:38.071Z" },
]
[[package]]
@@ -981,7 +924,7 @@ wheels = [
[[package]]
name = "litellm"
-version = "1.88.5"
+version = "1.93.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
@@ -997,9 +940,22 @@ dependencies = [
{ name = "tiktoken" },
{ name = "tokenizers" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/dc/ff/d8b95609ce0a45586cded6dfce7b657e2175eeb5e6b25cd27338bbc30765/litellm-1.88.5.tar.gz", hash = "sha256:a5f8a353ffb702a60c17bf0394596c0f051a34ef0998b313910074af137bbde6", size = 13886410, upload-time = "2026-06-24T23:56:57.744Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/97/dd/28024c0e4cf2dc6ab1bad59b8357af7f460e952c69526eae28f12ac4ee5e/litellm-1.93.2.tar.gz", hash = "sha256:c5d5223ef07f36e0886397fb45cc9db4150f86a0c6f6835cee1d5524cab69dfd", size = 15955441, upload-time = "2026-08-09T02:17:49.646Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/de/73/04f0b1ed787f3e7c7b000fa6a11d2ff6e264b9336581a83a4e196876d9ae/litellm-1.88.5-py3-none-any.whl", hash = "sha256:fcb86a76d934f9f228de7061c29d5632daf7e93e64ed09f81731cbdfa809346c", size = 15268212, upload-time = "2026-06-24T23:56:54.514Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/05/72fd8051f0f2f3c84b90986e6f4551db7c8b190ba3300f111461b7701689/litellm-1.93.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3bf532c164ad7cb1b76f2c62afefdcc656b9b296374d075a4150e2ce10bb74c3", size = 19937403, upload-time = "2026-08-09T02:16:55.545Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/4d/5081b39bdb73cab04f8a86294a4534a029cf0434ac6932c7ae8049d55723/litellm-1.93.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:526b7afc037f79dfdd5c607f5085ac597c7fd301a6dedabea40baae899b27f19", size = 19853652, upload-time = "2026-08-09T02:16:57.977Z" },
+ { url = "https://files.pythonhosted.org/packages/70/3f/fb70691266a7fd08c202406abea0153e82fa17f134cd9d58e4029cc741db/litellm-1.93.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:294ad19f356f821ce97a5428d09439be5f38d22b218c73008d8a49e3e42eb145", size = 20165680, upload-time = "2026-08-09T02:17:00.65Z" },
+ { url = "https://files.pythonhosted.org/packages/81/91/84424ce2a25595463e5d24e9cf8949877cd4ce93c0fcbf6486ecd685094f/litellm-1.93.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6f6a5e3907f0a1c9d8ff8d71a6cbac8a592e47a40da3f97167074947b5ba7d11", size = 20157772, upload-time = "2026-08-09T02:17:03.027Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/8d/b0eac7ee6d174564f820565c8c9a726ae83dbb8c4d3522daf175b95da002/litellm-1.93.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8541f1b7fd5c437ad249ad68d0a11f68e5e2866b0649da5fa7d63b595e9b8b22", size = 20229256, upload-time = "2026-08-09T02:17:05.271Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/6d/03e931c1cb2d1e1b7a968de21aa9e4db853928200da856c35c940ee6faa9/litellm-1.93.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:712c9387419d7b06a10df59973f5e530592d61b2314102b0fa3142f3743f9a9e", size = 20287257, upload-time = "2026-08-09T02:17:08.175Z" },
+ { url = "https://files.pythonhosted.org/packages/16/05/6c0fe2fcf31c260474c55fabe4ecb0e9e1343c9b9132e28589391b2ad33e/litellm-1.93.2-cp312-cp312-win_amd64.whl", hash = "sha256:cc0d58ccabd22ef7ef44a9e6f7247deb54ae42f5e126e6f00360c2b28b41bc2b", size = 19772580, upload-time = "2026-08-09T02:17:11.254Z" },
+ { url = "https://files.pythonhosted.org/packages/70/74/e9046cffa69b32b710452480598e418b26a29896ece680c80ec23997fd16/litellm-1.93.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f4071bef03e4c2942cd2ddc752727345b85447d6a7fee1ff5a4f8b92187966b0", size = 19938095, upload-time = "2026-08-09T02:17:13.929Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/db/6ef38a7a2f73d5cc507423954fa535a8546ead375c4c71265c093bdb4e9e/litellm-1.93.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8a99ac7c0c1b78acd6bfd1959e9f203dca71fdbceb5f0c8691c2ad8eee450d7d", size = 19854187, upload-time = "2026-08-09T02:17:16.588Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/b3/80ee0143b88e2921f8c8f24c7331478258a8bf25a3d4d4450bd96043403e/litellm-1.93.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a81ceff44c58ef504ab8bd787d03b82618765b9cfd530942386ae6d23c58be94", size = 20166307, upload-time = "2026-08-09T02:17:19.078Z" },
+ { url = "https://files.pythonhosted.org/packages/98/60/cb326e1094f7042f28f9e21543d9f367a8aa25af6915bf4253b77da5c2a2/litellm-1.93.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:dee1b02b7f52a5a408bf7c8d499f0834e49194651743758a511dcdd926c0b692", size = 20158336, upload-time = "2026-08-09T02:17:21.507Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/87/bad75146863531172c9dbae189486c7f4425b56a6641b55ab20745316048/litellm-1.93.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d2edfa14b99bce706b35981703692e3ee631f9b87bf6dc28fb53b574f6480b20", size = 20229711, upload-time = "2026-08-09T02:17:24.073Z" },
+ { url = "https://files.pythonhosted.org/packages/df/28/040b1853021ed8fd57be19eb2affb024d168951fe7e7abdbad91da3f6f3f/litellm-1.93.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ae75a61c9abc827aa3131b7e640c952367a450830bb7c531b426b4ec2bb45f85", size = 20287584, upload-time = "2026-08-09T02:17:26.542Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/0b/4208815b0d666636cbf7afbd571eec3004d3a15d3150a23a9009fc2ce930/litellm-1.93.2-cp313-cp313-win_amd64.whl", hash = "sha256:c54a09ab20f94120a9d60a30d9970439dcefa00d2565d190505ff006a80c7a69", size = 19772641, upload-time = "2026-08-09T02:17:29.308Z" },
]
[[package]]
@@ -1182,52 +1138,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" },
]
-[[package]]
-name = "multiprocess"
-version = "0.70.19"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "dill" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/e3/45/8004d1e6b9185c1a444d6b55ac5682acf9d98035e54386d967366035a03a/multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87", size = 134948, upload-time = "2026-01-19T06:47:32.325Z" },
- { url = "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", size = 144457, upload-time = "2026-01-19T06:47:33.711Z" },
- { url = "https://files.pythonhosted.org/packages/71/70/38998b950a97ea279e6bd657575d22d1a2047256caf707d9a10fbce4f065/multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28", size = 150281, upload-time = "2026-01-19T06:47:35.037Z" },
- { url = "https://files.pythonhosted.org/packages/7f/74/d2c27e03cb84251dfe7249b8e82923643c6d48fa4883b9476b025e7dc7eb/multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952", size = 156414, upload-time = "2026-01-19T06:47:35.915Z" },
- { url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" },
-]
-
-[[package]]
-name = "numpy"
-version = "2.5.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" },
- { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" },
- { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" },
- { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" },
- { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" },
- { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" },
- { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" },
- { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" },
- { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" },
- { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" },
- { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" },
- { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" },
- { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" },
- { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" },
- { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" },
- { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" },
- { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" },
- { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" },
- { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" },
- { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" },
- { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" },
- { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" },
-]
-
[[package]]
name = "openai"
version = "2.44.0"
@@ -1265,42 +1175,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" },
]
-[[package]]
-name = "pandas"
-version = "3.0.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "numpy" },
- { name = "python-dateutil" },
- { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09", size = 10357846, upload-time = "2026-05-11T18:52:36.143Z" },
- { url = "https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", size = 9899550, upload-time = "2026-05-11T18:52:38.976Z" },
- { url = "https://files.pythonhosted.org/packages/31/a8/fa2535168fffcedf67f4f6de28d2dd903a747ca7c8ea6989451aaeb3a92f/pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", size = 10412965, upload-time = "2026-05-11T18:52:41.915Z" },
- { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" },
- { url = "https://files.pythonhosted.org/packages/c9/a4/2eb28f2fccb4ced4a2c79ab2a5dee9ade1ebf44922ebad6fea158c9f95d4/pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", size = 11422824, upload-time = "2026-05-11T18:52:48.058Z" },
- { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" },
- { url = "https://files.pythonhosted.org/packages/b9/c5/fc1b368f303087d20e8c9bf3d6ceb186263cfac0ade735cd938538bea839/pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", size = 9755463, upload-time = "2026-05-11T18:52:53.386Z" },
- { url = "https://files.pythonhosted.org/packages/86/bd/fda8f9705b1b09c6ebe14bfc0fa0e4ec8584d54ea673628f157ff55131af/pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", size = 9066158, upload-time = "2026-05-11T18:52:56.038Z" },
- { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" },
- { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" },
- { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" },
- { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" },
- { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" },
- { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" },
- { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" },
- { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" },
- { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" },
- { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" },
- { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" },
- { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" },
- { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" },
- { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" },
- { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" },
-]
-
[[package]]
name = "pathspec"
version = "1.1.1"
@@ -1312,11 +1186,11 @@ wheels = [
[[package]]
name = "pip"
-version = "26.1.2"
+version = "26.2.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/01/91/47e7d486260f618783899587af63ccf7980fb60245c3e63dd4571c6b57ad/pip-26.1.2.tar.gz", hash = "sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605", size = 1840799, upload-time = "2026-05-31T17:33:58.56Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ae/15/4500e320e6b101ec3b719ae85b697d9940b6cda672bc555bd6016fc60c6f/pip-26.2.1.tar.gz", hash = "sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f", size = 1848877, upload-time = "2026-08-04T22:51:14.148Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/5d/95/6b5cb3461ea5673ba0995989746db58eb18b91b54dbf331e72f569540946/pip-26.1.2-py3-none-any.whl", hash = "sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab", size = 1813144, upload-time = "2026-05-31T17:33:56.772Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", hash = "sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e", size = 1816632, upload-time = "2026-08-04T22:51:12.472Z" },
]
[[package]]
@@ -1471,35 +1345,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9b/bf/7595e817906a29453ba4d99394e781b6fabe55d21f3c15d240f85dd06bb1/py_serializable-2.1.0-py3-none-any.whl", hash = "sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304", size = 23045, upload-time = "2025-07-21T09:56:46.848Z" },
]
-[[package]]
-name = "pyarrow"
-version = "24.0.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" },
- { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" },
- { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" },
- { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" },
- { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" },
- { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" },
- { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" },
- { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" },
- { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" },
- { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" },
- { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" },
- { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" },
- { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" },
- { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" },
- { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" },
- { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" },
- { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" },
- { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" },
- { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" },
- { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" },
- { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" },
-]
-
[[package]]
name = "pycparser"
version = "3.0"
@@ -2032,11 +1877,11 @@ requires-dist = [
{ name = "boto3", marker = "extra == 'llm'", specifier = ">=1.34.0" },
{ name = "build", marker = "extra == 'dev'", specifier = ">=1.2" },
{ name = "click", specifier = ">=8.3.3,<9" },
- { name = "harbor", marker = "extra == 'tier3'", specifier = "==0.13.2" },
+ { name = "harbor", marker = "extra == 'tier3'", specifier = "==0.22.0" },
{ name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27" },
{ name = "idna", specifier = ">=3.10,<4" },
{ name = "jinja2", specifier = ">=3.1" },
- { name = "litellm", marker = "extra == 'llm'", specifier = ">=1.83.10,<1.89.0.dev0" },
+ { name = "litellm", marker = "extra == 'llm'", specifier = ">=1.92.0,<1.94.0.dev0" },
{ name = "markdown-it-py", specifier = ">=4.2,<5" },
{ name = "mcp", marker = "extra == 'tier3'", specifier = ">=1.28.1,<2" },
{ name = "openai", marker = "extra == 'llm'", specifier = ">=2.21.0" },
@@ -2340,15 +2185,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" },
]
-[[package]]
-name = "tzdata"
-version = "2026.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" },
-]
-
[[package]]
name = "urllib3"
version = "2.7.0"
@@ -2402,76 +2238,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" },
]
-[[package]]
-name = "xxhash"
-version = "3.8.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/78/ed/07e560876a4458987511461187b285071f53cde49dd5b25cd8c51091522b/xxhash-3.8.0.tar.gz", hash = "sha256:d72b2204f37840b0f16f34192c09b994b97bd25823d723d47a1eddfacf06eb43", size = 86107, upload-time = "2026-06-27T08:17:28.798Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/17/2e/4b7c3ab28b7a54ac17eae7e02471c49609d6fc5900856a455feeb847a2a3/xxhash-3.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fc4bd14f873cd0b420f6f1ff5b5cd0dbfeb05b044a11bb9345bcbbf9749636e3", size = 34623, upload-time = "2026-06-27T08:13:16.696Z" },
- { url = "https://files.pythonhosted.org/packages/43/e4/09eea3e1bba6a59d64599cb8fba39f2a0872d06e85420eae989a4da61a9d/xxhash-3.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:31904979198e913239cb61b49f5b849696aeb3b03340da815d1491ec74dcc602", size = 32318, upload-time = "2026-06-27T08:13:18.036Z" },
- { url = "https://files.pythonhosted.org/packages/01/59/688bbae31e4e2d6d6eb92acbd3837c0e44ff8c7d435e6da922844ff6efda/xxhash-3.8.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7338ad13f2b273a1ef0ea97b2db0a059fdb3a1a29298bfa145937c0e4152d341", size = 220461, upload-time = "2026-06-27T08:13:19.311Z" },
- { url = "https://files.pythonhosted.org/packages/2d/de/71484ce0dab2fa4a475705d1ebc37a17ff02d40e5df6767b3255cc53120e/xxhash-3.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54e80e803cb34c8a1d278b491e543af40a588d288589c3e6becc991d5328b46b", size = 241110, upload-time = "2026-06-27T08:13:20.844Z" },
- { url = "https://files.pythonhosted.org/packages/bf/f9/1ac88f02e7df7898541490260b21f2b7f7bd2b233038a0cbd3a3b1bffdc2/xxhash-3.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:353953ea18f5c3fbdd13936fb536aacfb47d5bc06eef0919b1a355df61f7cc31", size = 264779, upload-time = "2026-06-27T08:13:22.485Z" },
- { url = "https://files.pythonhosted.org/packages/25/49/7ea1f128d2fe948ed679020f97a0896cdc6c975da5cc69b53a4a9c4a5def/xxhash-3.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d761f983a315630eff18c2fec7360c6b6946f82748026e779336eb8141ef3eba", size = 242609, upload-time = "2026-06-27T08:13:24.277Z" },
- { url = "https://files.pythonhosted.org/packages/a0/da/7d237278dfa1c48722c31010c84a328a317b8885429c8cb6ae4a8fa3e3db/xxhash-3.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f3786a9beb9a3b76241cb7db5f5388b460682c12204236389e3221963fc626a6", size = 473472, upload-time = "2026-06-27T08:13:25.877Z" },
- { url = "https://files.pythonhosted.org/packages/9b/5f/980fda82620a07d80026b4df371cbca12fca0fd94d7087c4ec5d898da76f/xxhash-3.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c94f5a9a775f36cc522fa2a7e8e2cec512e252d2ac056759f753dc68a79ffc", size = 220374, upload-time = "2026-06-27T08:13:27.366Z" },
- { url = "https://files.pythonhosted.org/packages/14/71/efa37bc3e91e1c801972bcef99eab877fcbd17ec10aca16c550ee2951107/xxhash-3.8.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:55ce59f9af37ac861947b43ea3ce7b294b5de77a1234b558d0f07ffad0197624", size = 310220, upload-time = "2026-06-27T08:13:28.804Z" },
- { url = "https://files.pythonhosted.org/packages/9d/48/19e40320044dc7051e8446505f18557d5661853b87a8770ad399325bb3c8/xxhash-3.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3afa1422a32c7c8e79ad5121dc21eaa5cee9e9e67bffca3f15d15d220d371908", size = 238100, upload-time = "2026-06-27T08:13:30.378Z" },
- { url = "https://files.pythonhosted.org/packages/d3/0d/588499f4d7cd064864ada7adfb9e8785f88a988f1332ed4c1be73d249c15/xxhash-3.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:551fda694938be910529452a89175137c58b4739e41fadff3c047e24b1d74a3b", size = 268937, upload-time = "2026-06-27T08:13:31.867Z" },
- { url = "https://files.pythonhosted.org/packages/54/18/fb2ad593572a33d1b6864b33047b8ca7269273a3c56107b5fd33e0b9c8fb/xxhash-3.8.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:512eb937c9457e6057e230e005c4709dd2ab63a5989f854d69f31db905750a62", size = 224910, upload-time = "2026-06-27T08:13:33.659Z" },
- { url = "https://files.pythonhosted.org/packages/63/9e/b880f9ed61b73492e24bb962d76aeb63f18ccb895f0edfb52e20d45ed6f2/xxhash-3.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4931ea93840f750a908efebaf23c71004feacc1a4649ef601b96d400a505c9a9", size = 240742, upload-time = "2026-06-27T08:13:35.237Z" },
- { url = "https://files.pythonhosted.org/packages/3f/89/fc682f93e54e486fc338b26a7d6d0d5cb0ab366269273c2608ac62b51afb/xxhash-3.8.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2fd4b60e8d9fc3923f39079f185b3425e6d76636fcb66d82a33dd7eba7c30f2f", size = 300527, upload-time = "2026-06-27T08:13:36.997Z" },
- { url = "https://files.pythonhosted.org/packages/80/71/a4b4122afb2d17ad69e0922cfeddb5ad5c25b02f37eed3dd3819d42e5f55/xxhash-3.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1da00075f1605794298878cb587f7533329693e2a0c45bbd25d6353644add675", size = 443195, upload-time = "2026-06-27T08:13:38.719Z" },
- { url = "https://files.pythonhosted.org/packages/d5/e5/ed3930f5dc90f4b1bab5ac3be099e8b2e81c1262d85e4adb5f2758e30d23/xxhash-3.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba73801c87d44fa37b2a5feab3004f0a654506027bf032ceb154d94bb74ea772", size = 217252, upload-time = "2026-06-27T08:13:41.179Z" },
- { url = "https://files.pythonhosted.org/packages/44/ae/128ea5794387ca54bb4084566db20dbdfc9c21cb17b67d3fcb403927b5ba/xxhash-3.8.0-cp312-cp312-win32.whl", hash = "sha256:0b0836dee6022e22ba516ebfa8f76c6e4bda08d6c166c553e40867bac89e4a54", size = 31890, upload-time = "2026-06-27T08:13:42.568Z" },
- { url = "https://files.pythonhosted.org/packages/4f/04/a6c182dc566c88e8d1a497d22cc4ffdcfcc0a9fa80325efa6cd4b9002c54/xxhash-3.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3bc2a09b98b8f85c75208cd2b2d2aecf40c77ecb2d72f6bf9757db51a98d3499", size = 32677, upload-time = "2026-06-27T08:13:43.705Z" },
- { url = "https://files.pythonhosted.org/packages/93/b5/aeda4e79f962c8d58ec60cb20a5abfe91c9f7d62e626f69f6659bc0bd0c4/xxhash-3.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:208e6a8b93426896d803224e9fabe26f8b9c651e8381a80b1fa31812faa091e3", size = 29155, upload-time = "2026-06-27T08:13:44.903Z" },
- { url = "https://files.pythonhosted.org/packages/ec/1f/96f43c5c7c7c4d44721f8d2e5d74698c667a30283c4b10a7e50a56804ee3/xxhash-3.8.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:36434c1d1b0a4729df1fa26ab11bffed1ba52666c0beb605c98a995b470cd143", size = 38508, upload-time = "2026-06-27T08:13:46.152Z" },
- { url = "https://files.pythonhosted.org/packages/1c/d9/7d5d6af4876c6481f2e0acb2dda64dd5209574bf7ba1ad4f6af7a1f8d473/xxhash-3.8.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:a5e6497cefcb2d67f1745c66df9718a99112583af6cc2b70da0312a2eb939f1e", size = 36542, upload-time = "2026-06-27T08:13:47.497Z" },
- { url = "https://files.pythonhosted.org/packages/32/ff/66fed439d78c5a09a1491a85af29bf8923b516530116731a9ac6b14dee2b/xxhash-3.8.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:5b00b82f1be708da9404fefd658cf5cf3be5ee3be2aae4bfe3b874255badd342", size = 31102, upload-time = "2026-06-27T08:13:48.721Z" },
- { url = "https://files.pythonhosted.org/packages/56/b8/9fae0399281095f8aca1f32b21947b3c3c75ad6021b255c5c6e4b11d3866/xxhash-3.8.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38b0cb0ab7f283413b7cace2bf710d7cf8f702ea82cbc683908691d52028a89b", size = 32096, upload-time = "2026-06-27T08:13:50.138Z" },
- { url = "https://files.pythonhosted.org/packages/61/a4/e53d162c74a8a2950dc063969914387b0680da4c7c20ad17744ec03a3b0a/xxhash-3.8.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:084312171a9798dea85e924b2674f5e1a44933050a1ea1cb1c6b1364e004c66c", size = 34585, upload-time = "2026-06-27T08:13:51.572Z" },
- { url = "https://files.pythonhosted.org/packages/69/f5/e12397e3f2c4917b6572e103a3277cd27cc56330e304bba61d195d7e5224/xxhash-3.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6a1a9e845bd3bbc57d9356819e0d198fe23282e0576b398a6282a0f8fdc75aef", size = 34622, upload-time = "2026-06-27T08:13:52.818Z" },
- { url = "https://files.pythonhosted.org/packages/70/80/c053dc51af5c942229689a0e9cb66fdc999bbd840f645e761f5ab73cbb17/xxhash-3.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9ffbde09743ebaf8957b8426948fbe85eab5e5de0d29eec407fcff5a2812a3cc", size = 32320, upload-time = "2026-06-27T08:13:54.04Z" },
- { url = "https://files.pythonhosted.org/packages/f8/a3/294171b67dfe770e1293edcf2a3f7e41302cdb8aefb258585312191b3ffe/xxhash-3.8.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a6dee3952c2b6e82e7f1dbc5dbc6167f9c84126851def7926e32827c2816169c", size = 220532, upload-time = "2026-06-27T08:13:55.448Z" },
- { url = "https://files.pythonhosted.org/packages/80/c3/d141bfdeca785c8c680abf867d4b52a5e64a55d90df242c3141a3e58c4b2/xxhash-3.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf8ff8e12416c9fa05b43c7509b9332d6ffc4090413c4e7a1dee8599763b6d59", size = 241215, upload-time = "2026-06-27T08:13:57.047Z" },
- { url = "https://files.pythonhosted.org/packages/09/5a/aeaf35143a6f3d44db73298e861405bdd9c9dacaedfc369cb43d9fd65282/xxhash-3.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cebbb322df4d97d8ef2704f49ed2f6f21f6702fafa0dc0c2a6ae70e904205689", size = 264615, upload-time = "2026-06-27T08:13:58.912Z" },
- { url = "https://files.pythonhosted.org/packages/bf/3e/f8ca782bb34f99693faab70a7989bcc84f62ffe93c9a4cca464a33507a4b/xxhash-3.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a8d08707b4100ebce598fc59fadf04b42d79b855818d6994f8f0fffd1df8edb", size = 242682, upload-time = "2026-06-27T08:14:00.483Z" },
- { url = "https://files.pythonhosted.org/packages/d4/fa/ddbee4ff1542c2e88e72269a5a6bd18c3f26a80c2514e0918f5d1f3e9ec5/xxhash-3.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cf5427602dda15d8ce3c6d870d29bf07d43975f59c9d6d3f7f6f93a901b28b12", size = 473551, upload-time = "2026-06-27T08:14:02.17Z" },
- { url = "https://files.pythonhosted.org/packages/25/f5/a680d48dddab37ab2fd9189ca03f775e29e3627122e30790816d7eb365af/xxhash-3.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97d7bd715ea5050b6c9638b52c62adf3055b648ef6eee6892a4cd9697b530191", size = 220485, upload-time = "2026-06-27T08:14:03.765Z" },
- { url = "https://files.pythonhosted.org/packages/22/b1/7ac129b74981c07f1ff9c649f204465e86f83f9f29b2ebdc70d91514c365/xxhash-3.8.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0cd25bbbab37d898f6e5a90905ce6ae2c1f8bd6668c07cef406fb3e8c8c570dd", size = 310307, upload-time = "2026-06-27T08:14:05.366Z" },
- { url = "https://files.pythonhosted.org/packages/67/e6/43e673411249dd63f6cd974523a1b32fad75cf5453e363bc8f44af215fb9/xxhash-3.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3e30e5c057f483c3c53a11b53eba091a737cb19dfead36c8b23bf5beb4a169cd", size = 238164, upload-time = "2026-06-27T08:14:07.149Z" },
- { url = "https://files.pythonhosted.org/packages/e5/95/87f8baf41f63130f3637104b7a610f82b20106332fc6e289c8dbf7955d0e/xxhash-3.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:07dd44d992ebd456752bc25b1c42cd172d94bd8cb24049300449ad0716081c3a", size = 269062, upload-time = "2026-06-27T08:14:08.834Z" },
- { url = "https://files.pythonhosted.org/packages/38/c9/3369b497cd1f926b930c52fd2400606f177790d887b49f9e86bddcc24562/xxhash-3.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:3118600a3102d4707dc1c485dbc3acbbbf37819069ad3e7854e77b923745d76b", size = 225007, upload-time = "2026-06-27T08:14:10.689Z" },
- { url = "https://files.pythonhosted.org/packages/34/c8/03dceb86a8128858ac105bd6e282d62b3db6fd421a79bd8a9f6b8cdc47a7/xxhash-3.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7ed37b0c95d8fb3fbaad5e13cc0a9727eb8739d1d54b2adef28108c250cada3a", size = 240815, upload-time = "2026-06-27T08:14:12.195Z" },
- { url = "https://files.pythonhosted.org/packages/47/a5/ebd43eeb1af1dd8f0201943688b20958e99d3f6eb36481fb8c37b55ef139/xxhash-3.8.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bb043da412e478e7b1db3407051124b85b133803794d3809ad6d92870b304fc7", size = 300632, upload-time = "2026-06-27T08:14:13.916Z" },
- { url = "https://files.pythonhosted.org/packages/df/24/c873e41a3c00dacc385c8ff08c007723f6a528922c1cea7fd9684e86dae7/xxhash-3.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:196fc132683d9311a0bdce8388ee52bfa07fdc1987cc428a27956e47ccd7b50d", size = 443293, upload-time = "2026-06-27T08:14:15.446Z" },
- { url = "https://files.pythonhosted.org/packages/4f/1b/c671272fe28f70574e3c574d58465f26460154bcc68876121872afa1c14d/xxhash-3.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfb5411af3b77c75e99db100aa15c5ba623c85d72c565e4d7a0ed1a986ff766e", size = 217327, upload-time = "2026-06-27T08:14:17.28Z" },
- { url = "https://files.pythonhosted.org/packages/57/43/b45a52f795812cb769b6ac159e69b605d18b1c067749e63dcac159e90064/xxhash-3.8.0-cp313-cp313-win32.whl", hash = "sha256:6d1d6179e26830c6690fac63f76d372f69714b977e12ca9c42188a60f51c59f5", size = 31898, upload-time = "2026-06-27T08:14:18.952Z" },
- { url = "https://files.pythonhosted.org/packages/a1/42/2bd70e4eec25dc5990652979d708d4d7c999793d7d5af5d0e48ab4374dc1/xxhash-3.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:7c92427a56a12f4d5c7bb26dbb9e9a4658c313ecb6c2f1dca349902e3822df07", size = 32680, upload-time = "2026-06-27T08:14:20.277Z" },
- { url = "https://files.pythonhosted.org/packages/d5/c8/2fe61edb6144183cf094035a8c5354c65a073127acf6379655ed1e705b70/xxhash-3.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:9fc8453642c1c6d38b4fbac8901c2452ce1fa88b27f003bfee6703cbfae9bd63", size = 29157, upload-time = "2026-06-27T08:14:21.674Z" },
- { url = "https://files.pythonhosted.org/packages/b9/b8/81d17a993b9a4750ba426ce966421681bb4b8e82a460cd346756491b8cc2/xxhash-3.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:efcacb644a915f010dc477447b045e5dcde1afaa40d16b2f0f8e7cd99c9e1635", size = 34897, upload-time = "2026-06-27T08:14:23.044Z" },
- { url = "https://files.pythonhosted.org/packages/bb/3b/f5a368e3273440b3ea58fbd3f0b08c19f552b25ca59f43f5732ca96d2126/xxhash-3.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d1e0dbc510cff94c5efbcc2b82c28b41519fad09b5b1f9f3d99c63e3940e49a0", size = 32630, upload-time = "2026-06-27T08:14:24.603Z" },
- { url = "https://files.pythonhosted.org/packages/a6/ab/f424359c91c55f564fbbe4e454a126eb522471109f67376f20ad19c5e663/xxhash-3.8.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ff19d016a41c90d1f519005887191896b6da1274e1d5d48b347e17eb798ffc5a", size = 225874, upload-time = "2026-06-27T08:14:25.992Z" },
- { url = "https://files.pythonhosted.org/packages/ac/c2/434579ef9235123b6c9bfa89c5614e0001e988613b91557b24aa326d9faa/xxhash-3.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aafc3eab99c50508852e34307e9565933bf128cad084cac7d2471b7ab1743de0", size = 249705, upload-time = "2026-06-27T08:14:27.607Z" },
- { url = "https://files.pythonhosted.org/packages/c0/6c/3c0c917331ca3c71f826cedce2127f230624e2b49b992472dd5e9e72101c/xxhash-3.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5e521368ed79ae6c4d31e1e417726643c49d7d6e286f4fdabf9a8330ed8a8ff7", size = 274716, upload-time = "2026-06-27T08:14:29.495Z" },
- { url = "https://files.pythonhosted.org/packages/c1/f3/a8bb98d3307c67e88be9642dff52854c3de3f488f95989b60ff69c8dcc42/xxhash-3.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6a0127688d116ec0c225e7e1f744e3f206de2b8822ffeb31a9ab5cc6384f92c5", size = 252019, upload-time = "2026-06-27T08:14:31.247Z" },
- { url = "https://files.pythonhosted.org/packages/f7/73/fab69a2e5b6353dde643209fe9b6adf4fbd64c888e531deffc476bfb2635/xxhash-3.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:22c0b17da2f9fea0f8836538512249871b359141616bad44c58d238b5f011f40", size = 482024, upload-time = "2026-06-27T08:14:32.973Z" },
- { url = "https://files.pythonhosted.org/packages/a3/5b/ba34099b5278097ec9c68c0b740719813553bfd11ca17e7353de6d2a41e3/xxhash-3.8.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d49465646b1a5e3b1729c5f636e05676a2fb52e203e3b22a5411c416c4c5302", size = 226655, upload-time = "2026-06-27T08:14:34.608Z" },
- { url = "https://files.pythonhosted.org/packages/76/0c/90aba4708a37fe752b324a7cbf10058eaa33e892cdd62751ff17a5137b93/xxhash-3.8.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c2853dea1e30ed00ca87dd87d76da5da063d302b823b3fb80ccd18421de0f251", size = 319583, upload-time = "2026-06-27T08:14:36.419Z" },
- { url = "https://files.pythonhosted.org/packages/38/46/42e349e2d3017b2688f4cb301742c37c438e77963e3fef711edce2fc5c65/xxhash-3.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:82f0102a2a3760287b7cd7f9e0a30edd4c3b18762ed1a242208d43c8e2bcf30b", size = 246000, upload-time = "2026-06-27T08:14:38.104Z" },
- { url = "https://files.pythonhosted.org/packages/ee/15/741b947ae3c768e82018c46846f8616f6aa9b5042649f318a1a6897defe3/xxhash-3.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:b8414a66a7524596d841cad5dc1adab6ce76848db5ab2b83db911fbdab1417af", size = 275455, upload-time = "2026-06-27T08:14:39.841Z" },
- { url = "https://files.pythonhosted.org/packages/c6/b4/a9db84c9458fc8f53eaf0051377d1e9eecd9f330fb1225640027417a309d/xxhash-3.8.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:0dbaa73df10414ea1e41b98691a9d8241d4c47ad8d02c726587a3cda05278e53", size = 231209, upload-time = "2026-06-27T08:14:41.543Z" },
- { url = "https://files.pythonhosted.org/packages/20/92/60a868cd34851746d0b0d95dced0f42867c7c00606f6e5dba85b70b232ce/xxhash-3.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:43fc9aaba10ab4267c90793601f60d35c3c9caa1544eceb483618a71ad9ce7da", size = 250416, upload-time = "2026-06-27T08:14:43.193Z" },
- { url = "https://files.pythonhosted.org/packages/7a/6a/168ca46a4679c32aae9246caa1fddf35981d6304487e45e992b3d4530324/xxhash-3.8.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ec5eb3d28fbb9802c6d2526f772133a06c91d6f03756fcc67c834b642ffdd51d", size = 309764, upload-time = "2026-06-27T08:14:44.79Z" },
- { url = "https://files.pythonhosted.org/packages/18/0b/13646b348c07679c818791ab2d35415db5cb20f3bc77daaa255909a401b4/xxhash-3.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:2b77c301b644cd9b4d0749a3291081ec2048a6bef7fe0487c993bbba3efb9ce0", size = 448650, upload-time = "2026-06-27T08:14:46.562Z" },
- { url = "https://files.pythonhosted.org/packages/59/9a/3d244b2acf6bbd86a363817ee09084b4684e8e11840663e19869e9e0d952/xxhash-3.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d7ece11a132325353890a144c30119073617a1299c593ca29b96c315b07e1edd", size = 223572, upload-time = "2026-06-27T08:14:48.294Z" },
- { url = "https://files.pythonhosted.org/packages/6f/c7/143410d026a6e0d86dc69037ec2a3b8db810a54e7f443b340ac17612be2e/xxhash-3.8.0-cp313-cp313t-win32.whl", hash = "sha256:b21db84df7b9d54d9e4195a964243c1b32d745c6fbc0cfcfffee1d4bd297196a", size = 32301, upload-time = "2026-06-27T08:14:49.687Z" },
- { url = "https://files.pythonhosted.org/packages/6c/db/2240b0638161637b2f310231748a7a6a06c79fb43a3adb34c96f359762bf/xxhash-3.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:0643b7d9f598f6da6f1f6b899f4358250d0fb853242e2d712cbde27bf5a99d29", size = 33221, upload-time = "2026-06-27T08:14:51.404Z" },
- { url = "https://files.pythonhosted.org/packages/e0/d8/52038e4fa5baf4f00654a225516168d02908edfec7ca104fbefc58af394f/xxhash-3.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:4bbacf2e938526969f8ab3334d4ac3da14ea059e1dfd1339a92f9091467e750f", size = 29294, upload-time = "2026-06-27T08:14:52.778Z" },
-]
-
[[package]]
name = "yarl"
version = "1.24.2"