diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c4e84..ea927b43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to SkillEvaluator are documented in this file. ## Unreleased +### Added + +- Added an append-only, vendor-neutral error-code registry for evaluator + execution failures. Terminal provider and runtime-preflight results now + include a structured `error_code` while preserving `execution_errors` and + existing lowercase provider failure subtypes for compatibility. + ### Fixed - Tier 3 accuracy and custom goal judges now retry one malformed (including diff --git a/docs/README.md b/docs/README.md index 4909e260..ad53df5e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,7 @@ contributors making non-trivial edits. - `tier1-validation.mdx` — checks, flags, reports, CI recipe - `tier2-deduplication.mdx` — dedup commands and thresholds - `tier3-live-evaluation.mdx` — skill evaluation with live agents +- `error-codes.mdx` — stable execution failure identifiers - `developer-guide.mdx` — contributor setup Navigation order and slugs are defined in [`../fern/docs.yml`](../fern/docs.yml). diff --git a/docs/error-codes.mdx b/docs/error-codes.mdx new file mode 100644 index 00000000..3bba88a9 --- /dev/null +++ b/docs/error-codes.mdx @@ -0,0 +1,83 @@ +--- +title: "Error Code Reference" +description: "Stable, vendor-neutral error codes for SkillEvaluator execution failures." +--- + +SkillEvaluator emits `error_code` as the stable support-facing identifier for a +terminal evaluator execution failure. Automation should use this field instead +of matching human-readable messages. + +```json title="Failed evaluation result" +{ + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-AUTH-002", + "execution_errors": ["Provider verification failed."] +} +``` + +`execution_errors` remains available for backward compatibility and human +diagnosis. Lowercase fields such as `failure_kind`, when present on a provider +probe, are diagnostic subtypes rather than stable support identifiers. Their +values may be more implementation-specific than `error_code`. + +## Contract + +Codes match this uppercase ASCII schema: + +```text +^SKILLEVALUATOR-[A-Z][A-Z0-9]*-[0-9]{3}$ +``` + +The in-package registry is authoritative and append-only. A released code is +never renamed, reused, or assigned a different meaning. A string that matches +the schema but is absent from the registry is not a valid SkillEvaluator error +code. + +Python integrations can obtain a serialization-ready JSON Schema with +`skillevaluator.error_codes.error_code_schema()`. Each call returns a fresh +dictionary; changing it does not modify the authoritative registry. + +When several failures end one evaluation, `error_code` keeps their code only +when every classified failure agrees. Conflicting or absent classifications +produce `SKILLEVALUATOR-UNKNOWN-001`. `execution_errors` retains the available +human-readable details. Older result files may omit `error_code`; readers +continue to accept those files for compatibility. + +## Registered codes + +| Code | Meaning | +| --- | --- | +| `SKILLEVALUATOR-AUTH-002` | Provider authentication failed | +| `SKILLEVALUATOR-AUTH-003` | Provider authorization failed | +| `SKILLEVALUATOR-CONFIG-001` | Evaluator configuration was invalid | +| `SKILLEVALUATOR-DEPENDENCY-001` | A required dependency was unavailable | +| `SKILLEVALUATOR-DEPENDENCY-003` | The configured model was not found | +| `SKILLEVALUATOR-DEPENDENCY-005` | A required dependency timed out | +| `SKILLEVALUATOR-DEPENDENCY-006` | A required dependency rate-limited the evaluator | +| `SKILLEVALUATOR-DEPENDENCY-007` | The selected dependency operation is unsupported | +| `SKILLEVALUATOR-DEPENDENCY-008` | A required dependency returned an invalid response | +| `SKILLEVALUATOR-DEPENDENCY-009` | A required dependency returned another HTTP failure | +| `SKILLEVALUATOR-UNKNOWN-001` | The evaluator could not classify the failure | +| `SKILLEVALUATOR-RUNTIME-005` | The evaluator could not start a runtime process | +| `SKILLEVALUATOR-RUNTIME-007` | Evaluator runtime execution timed out | +| `SKILLEVALUATOR-RUNTIME-008` | An evaluator runtime process exited unsuccessfully | +| `SKILLEVALUATOR-CONTRACT-007` | The evaluator runtime produced an invalid job result | + +Provider and runtime preflight classifications use structured failure metadata +such as the failure subtype and HTTP status. They never infer a code from error +message text. In particular, an unavailable provider response maps as follows: + +| Structured signal | Code | +| --- | --- | +| HTTP 429 | `SKILLEVALUATOR-DEPENDENCY-006` | +| HTTP 408 or an explicit timeout | `SKILLEVALUATOR-DEPENDENCY-005` | +| HTTP 5xx or no more specific signal | `SKILLEVALUATOR-DEPENDENCY-001` | + +Local runtime-preflight failures use the `RUNTIME` domain. A process spawn +failure, execution timeout, or unsuccessful exit is not classified as a +dependency failure. This includes local credential-helper process spawn and +timeout failures. A missing or invalid runtime job result uses the `CONTRACT` +domain because the process ran but did not satisfy the result contract. + +See [Reports & Results](reports.mdx#machine-readable-contract) for the result +artifacts that carry this field. diff --git a/docs/reports.mdx b/docs/reports.mdx index e6466eb6..ab332874 100644 --- a/docs/reports.mdx +++ b/docs/reports.mdx @@ -311,6 +311,9 @@ For CI and tooling, two JSON entry points matter: - **Standalone `tier3 evaluate` runs** — parse `result.json` in the run directory (or follow `latest`). It carries per-agent scores, dimensions, lift, pass@k, trial counts, the attempt policy, and execution status. + Failed execution results also carry the stable `error_code`; use the + [Error Code Reference](error-codes.mdx) instead of matching prose in + `execution_errors`. Unscored metrics stay unscored rather than defaulting to 0.0, so `overall_score` can be `null`. - **`validate --tier3` runs** — the combined diff --git a/fern/docs.yml b/fern/docs.yml index 88e4ff16..4c5052d6 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -65,6 +65,9 @@ navigation: - page: Reports & Results path: ../docs/reports.mdx slug: reports + - page: Error Code Reference + path: ../docs/error-codes.mdx + slug: error-codes - page: "BENCHMARK.md Rollout and Backfill" path: ../docs/benchmark-rollout.mdx slug: benchmark-rollout diff --git a/src/skillevaluator/error_codes.py b/src/skillevaluator/error_codes.py new file mode 100644 index 00000000..2367dc56 --- /dev/null +++ b/src/skillevaluator/error_codes.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stable, vendor-neutral error codes for evaluator execution failures.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from enum import StrEnum +from types import MappingProxyType + + +class ErrorDomain(StrEnum): + """Factual subsystem that produced an evaluator execution failure.""" + + AUTH = "AUTH" + CONFIG = "CONFIG" + DEPENDENCY = "DEPENDENCY" + UNKNOWN = "UNKNOWN" + RUNTIME = "RUNTIME" + CONTRACT = "CONTRACT" + + +class EvaluatorErrorCode(StrEnum): + """Stable support-facing identifiers emitted by SkillEvaluator. + + Allocations are append-only. Existing values must never be renamed, reused, + or assigned a different meaning. + """ + + AUTHENTICATION = "SKILLEVALUATOR-AUTH-002" + AUTHORIZATION = "SKILLEVALUATOR-AUTH-003" + INVALID_CONFIGURATION = "SKILLEVALUATOR-CONFIG-001" + DEPENDENCY_UNAVAILABLE = "SKILLEVALUATOR-DEPENDENCY-001" + MODEL_NOT_FOUND = "SKILLEVALUATOR-DEPENDENCY-003" + DEPENDENCY_TIMEOUT = "SKILLEVALUATOR-DEPENDENCY-005" + RATE_LIMITED = "SKILLEVALUATOR-DEPENDENCY-006" + UNSUPPORTED = "SKILLEVALUATOR-DEPENDENCY-007" + INVALID_RESPONSE = "SKILLEVALUATOR-DEPENDENCY-008" + OTHER_HTTP = "SKILLEVALUATOR-DEPENDENCY-009" + UNKNOWN = "SKILLEVALUATOR-UNKNOWN-001" + PROCESS_SPAWN_FAILED = "SKILLEVALUATOR-RUNTIME-005" + EXECUTION_TIMEOUT = "SKILLEVALUATOR-RUNTIME-007" + PROCESS_EXITED = "SKILLEVALUATOR-RUNTIME-008" + JOB_RESULT_INVALID = "SKILLEVALUATOR-CONTRACT-007" + + +@dataclass(frozen=True, slots=True) +class ErrorCodeDefinition: + """Immutable public metadata for one registered error code.""" + + domain: ErrorDomain + summary: str + + +# Registry insertion order is allocation order. Add new entries at the end. +ERROR_CODE_REGISTRY: Mapping[str, ErrorCodeDefinition] = MappingProxyType( + { + EvaluatorErrorCode.AUTHENTICATION.value: ErrorCodeDefinition( + ErrorDomain.AUTH, + "Provider authentication failed.", + ), + EvaluatorErrorCode.AUTHORIZATION.value: ErrorCodeDefinition( + ErrorDomain.AUTH, + "Provider authorization failed.", + ), + EvaluatorErrorCode.INVALID_CONFIGURATION.value: ErrorCodeDefinition( + ErrorDomain.CONFIG, + "Evaluator configuration was invalid.", + ), + EvaluatorErrorCode.DEPENDENCY_UNAVAILABLE.value: ErrorCodeDefinition( + ErrorDomain.DEPENDENCY, + "A required dependency was unavailable.", + ), + EvaluatorErrorCode.MODEL_NOT_FOUND.value: ErrorCodeDefinition( + ErrorDomain.DEPENDENCY, + "The configured model was not found.", + ), + EvaluatorErrorCode.DEPENDENCY_TIMEOUT.value: ErrorCodeDefinition( + ErrorDomain.DEPENDENCY, + "A required dependency timed out.", + ), + EvaluatorErrorCode.RATE_LIMITED.value: ErrorCodeDefinition( + ErrorDomain.DEPENDENCY, + "A required dependency rate-limited the evaluator.", + ), + EvaluatorErrorCode.UNSUPPORTED.value: ErrorCodeDefinition( + ErrorDomain.DEPENDENCY, + "The selected dependency operation is unsupported.", + ), + EvaluatorErrorCode.INVALID_RESPONSE.value: ErrorCodeDefinition( + ErrorDomain.DEPENDENCY, + "A required dependency returned an invalid response.", + ), + EvaluatorErrorCode.OTHER_HTTP.value: ErrorCodeDefinition( + ErrorDomain.DEPENDENCY, + "A required dependency returned another HTTP failure.", + ), + EvaluatorErrorCode.UNKNOWN.value: ErrorCodeDefinition( + ErrorDomain.UNKNOWN, + "The evaluator could not classify the failure.", + ), + EvaluatorErrorCode.PROCESS_SPAWN_FAILED.value: ErrorCodeDefinition( + ErrorDomain.RUNTIME, + "The evaluator could not start a runtime process.", + ), + EvaluatorErrorCode.EXECUTION_TIMEOUT.value: ErrorCodeDefinition( + ErrorDomain.RUNTIME, + "Evaluator runtime execution timed out.", + ), + EvaluatorErrorCode.PROCESS_EXITED.value: ErrorCodeDefinition( + ErrorDomain.RUNTIME, + "An evaluator runtime process exited unsuccessfully.", + ), + EvaluatorErrorCode.JOB_RESULT_INVALID.value: ErrorCodeDefinition( + ErrorDomain.CONTRACT, + "The evaluator runtime produced an invalid job result.", + ), + } +) + +ERROR_CODE_PATTERN_TEXT = r"^SKILLEVALUATOR-[A-Z][A-Z0-9]*-[0-9]{3}$" +ERROR_CODE_PATTERN = re.compile(ERROR_CODE_PATTERN_TEXT, flags=re.ASCII) + + +def error_code_schema() -> dict[str, object]: + """Return a serialization-ready JSON Schema for registered error codes.""" + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SkillEvaluator error code", + "type": "string", + "pattern": ERROR_CODE_PATTERN_TEXT, + "enum": list(ERROR_CODE_REGISTRY), + } + + +def _validate_registry() -> None: + enum_values = {member.value for member in EvaluatorErrorCode} + if set(ERROR_CODE_REGISTRY) != enum_values: + raise RuntimeError("error code registry and enum are inconsistent") + for code, definition in ERROR_CODE_REGISTRY.items(): + if not code.isascii() or ERROR_CODE_PATTERN.fullmatch(code) is None: + raise RuntimeError(f"invalid registered error code: {code!r}") + if f"-{definition.domain.value}-" not in code: + raise RuntimeError(f"error code domain does not match its registry entry: {code}") + + +_validate_registry() + + +def is_registered_error_code(value: object) -> bool: + """Return whether *value* is an exact registered error code.""" + return isinstance(value, str) and value.isascii() and value in ERROR_CODE_REGISTRY + + +def validate_error_code(value: object) -> str: + """Return a valid registered code or raise ``ValueError``.""" + if not is_registered_error_code(value): + raise ValueError(f"unregistered SkillEvaluator error code: {value!r}") + return str(value) + + +def provider_failure_error_code( + failure_kind: object, + http_status: object = None, + *, + timed_out: bool = False, +) -> EvaluatorErrorCode: + """Map structured provider failure metadata without inspecting message text.""" + kind = str(failure_kind) if failure_kind is not None else "unknown" + status = http_status if isinstance(http_status, int) and not isinstance(http_status, bool) else None + if kind == "local_process": + return EvaluatorErrorCode.EXECUTION_TIMEOUT if timed_out else EvaluatorErrorCode.PROCESS_SPAWN_FAILED + if kind == "unavailable": + if status == 429: + return EvaluatorErrorCode.RATE_LIMITED + if timed_out or status == 408: + return EvaluatorErrorCode.DEPENDENCY_TIMEOUT + return EvaluatorErrorCode.DEPENDENCY_UNAVAILABLE + return { + "authentication": EvaluatorErrorCode.AUTHENTICATION, + "authorization": EvaluatorErrorCode.AUTHORIZATION, + "invalid_configuration": EvaluatorErrorCode.INVALID_CONFIGURATION, + "model_not_found": EvaluatorErrorCode.MODEL_NOT_FOUND, + "unsupported": EvaluatorErrorCode.UNSUPPORTED, + "invalid_response": EvaluatorErrorCode.INVALID_RESPONSE, + "other_http": EvaluatorErrorCode.OTHER_HTTP, + "unknown": EvaluatorErrorCode.UNKNOWN, + }.get(kind, EvaluatorErrorCode.UNKNOWN) + + +def primary_error_code(codes: Iterable[object]) -> str: + """Return a consensus code, or UNKNOWN when failures conflict or are absent.""" + registered = {str(code) for code in codes if is_registered_error_code(code)} + return registered.pop() if len(registered) == 1 else EvaluatorErrorCode.UNKNOWN.value + + +__all__ = ( + "ERROR_CODE_PATTERN", + "ERROR_CODE_PATTERN_TEXT", + "ERROR_CODE_REGISTRY", + "ErrorCodeDefinition", + "ErrorDomain", + "EvaluatorErrorCode", + "error_code_schema", + "is_registered_error_code", + "primary_error_code", + "provider_failure_error_code", + "validate_error_code", +) diff --git a/src/skillevaluator/model_catalog.py b/src/skillevaluator/model_catalog.py index bc60b774..88a7c2d4 100644 --- a/src/skillevaluator/model_catalog.py +++ b/src/skillevaluator/model_catalog.py @@ -24,6 +24,8 @@ from urllib.parse import quote, urlencode, urlsplit from urllib.request import HTTPHandler, HTTPRedirectHandler, HTTPSHandler, ProxyHandler, Request, build_opener +from skillevaluator.error_codes import EvaluatorErrorCode, provider_failure_error_code, validate_error_code + if TYPE_CHECKING: from skillevaluator.provider_config import ProviderConfig @@ -63,6 +65,7 @@ class ModelCatalogFailureKind(StrEnum): MODEL_NOT_FOUND = "model_not_found" OTHER_HTTP = "other_http" UNKNOWN = "unknown" + LOCAL_PROCESS = "local_process" class ModelCatalogError(RuntimeError): @@ -74,6 +77,8 @@ def __init__( *, kind: ModelCatalogFailureKind | str = ModelCatalogFailureKind.UNKNOWN, http_status: int | None = None, + error_code: EvaluatorErrorCode | str | None = None, + timed_out: bool = False, ) -> None: super().__init__(message) try: @@ -81,6 +86,11 @@ def __init__( except (TypeError, ValueError): self.kind = ModelCatalogFailureKind.UNKNOWN self.http_status = http_status + normalized_timed_out = bool(timed_out) + expected_code = provider_failure_error_code(self.kind, http_status, timed_out=normalized_timed_out).value + if error_code is not None and validate_error_code(error_code) != expected_code: + raise ValueError("catalog error code contradicts structured failure metadata") + self.error_code = expected_code def _http_failure_kind(status: int) -> ModelCatalogFailureKind: @@ -358,6 +368,7 @@ def fetch_model_records(config: ProviderConfig, timeout_seconds: float = 15.0) - raise ModelCatalogError( "model catalog request timed out", kind=ModelCatalogFailureKind.UNAVAILABLE, + timed_out=True, ) payload, response_bytes = _request_json( next_url, @@ -634,12 +645,14 @@ def _request_json( raise ModelCatalogError( "model catalog request timed out", kind=ModelCatalogFailureKind.UNAVAILABLE, + timed_out=True, ) from None except URLError as exc: if isinstance(exc.reason, TimeoutError): raise ModelCatalogError( "model catalog request timed out", kind=ModelCatalogFailureKind.UNAVAILABLE, + timed_out=True, ) from None raise ModelCatalogError( f"model catalog request failed: {type(exc).__name__}", @@ -690,6 +703,7 @@ def _read_response_body(response: Any, *, max_response_bytes: int, deadline: flo raise ModelCatalogError( "model catalog request timed out", kind=ModelCatalogFailureKind.UNAVAILABLE, + timed_out=True, ) return raw @@ -701,6 +715,7 @@ def _read_response_body(response: Any, *, max_response_bytes: int, deadline: flo raise ModelCatalogError( "model catalog request timed out", kind=ModelCatalogFailureKind.UNAVAILABLE, + timed_out=True, ) _set_response_socket_timeout(response, remaining) chunk = read_once(min(_RESPONSE_READ_CHUNK_BYTES, max_response_bytes + 1 - total)) @@ -708,6 +723,7 @@ def _read_response_body(response: Any, *, max_response_bytes: int, deadline: flo raise ModelCatalogError( "model catalog request timed out", kind=ModelCatalogFailureKind.UNAVAILABLE, + timed_out=True, ) if not chunk: break diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py index ff6d1b7d..4169b96e 100644 --- a/src/skillevaluator/tier3/harbor/runner.py +++ b/src/skillevaluator/tier3/harbor/runner.py @@ -29,6 +29,13 @@ from uuid import uuid4 from skillevaluator import __version__ +from skillevaluator.error_codes import ( + EvaluatorErrorCode, + is_registered_error_code, + primary_error_code, + provider_failure_error_code, + validate_error_code, +) from skillevaluator.evaluation.tier3_report import render_agent_eval_html_report from skillevaluator.provider_config import ( ProviderConfig, @@ -1774,6 +1781,20 @@ def _existing_file(path: str | None) -> str | None: return path if path is not None and Path(path).is_file() else None +def _terminal_failure_result( + errors: list[str], + *, + error_code: EvaluatorErrorCode | str, +) -> dict[str, Any]: + """Build the common serializable shape for an evaluator execution failure.""" + return { + "execution_status": "failed", + "error_code": validate_error_code(error_code), + "execution_errors": errors, + "error": errors, + } + + def _run_harbor_eval_impl( skill_path: Path, agents: list[str], @@ -1811,10 +1832,16 @@ def _run_harbor_eval_impl( reporter = safe_progress_reporter(progress_reporter or NullProgressReporter()) if env_mode not in HARBOR_ENV_MODES: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="unsupported environment")) - return {"error": [f"env_mode must be one of: {', '.join(sorted(HARBOR_ENV_MODES))}"]} + return _terminal_failure_result( + [f"env_mode must be one of: {', '.join(sorted(HARBOR_ENV_MODES))}"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if not agents: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="no agents selected")) - return {"error": ["At least one Harbor agent is required."]} + return _terminal_failure_result( + ["At least one Harbor agent is required."], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if env_mode == ENV_MODE_LOCAL: from skillevaluator.tier3.harbor import local_sandbox @@ -1822,7 +1849,7 @@ def _run_harbor_eval_impl( local_sandbox.require_supported_platform() except local_sandbox.SandboxUnavailable as exc: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=str(exc))) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.UNKNOWN) if _evaluator_skill_path is None: assert forwarded is not None @@ -1833,7 +1860,7 @@ def _run_harbor_eval_impl( evaluator_skill_path = snapshot_stack.enter_context(private_evaluator_skill_snapshot(skill_path)) except (OSError, ValueError) as exc: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=str(exc))) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.UNKNOWN) forwarded["_evaluator_skill_path"] = evaluator_skill_path forwarded["_monotonic_start"] = started_at return _run_harbor_eval_impl(skill_path, agents, **forwarded) @@ -1845,7 +1872,7 @@ def _run_harbor_eval_impl( config, config_path = load_evals_config(evaluator_skill_path) except (ProviderConfigurationError, EvalsConfigError) as exc: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail=str(exc))) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.INVALID_CONFIGURATION) harbor_config = config.get("harbor", {}) workspace_config = config.get("skill_workspace", {}) @@ -1873,25 +1900,46 @@ def _run_harbor_eval_impl( if not isinstance(n_attempts, int) or n_attempts < 1: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid attempt count")) - return {"error": ["n_attempts must be >= 1"]} + return _terminal_failure_result( + ["n_attempts must be >= 1"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if stop_on_pass and n_attempts == 1: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid attempt policy")) - return {"error": ["stop_on_pass requires n_attempts > 1"]} + return _terminal_failure_result( + ["stop_on_pass requires n_attempts > 1"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if not isinstance(n_concurrent, int) or n_concurrent < 1: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid concurrency")) - return {"error": ["n_concurrent must be >= 1"]} + return _terminal_failure_result( + ["n_concurrent must be >= 1"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if not isinstance(max_agents, int) or max_agents < 1: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid agent concurrency")) - return {"error": ["max_agents must be >= 1"]} + return _terminal_failure_result( + ["max_agents must be >= 1"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if not isinstance(pass_threshold, (int, float)) or not 0 <= float(pass_threshold) <= 1: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid pass threshold")) - return {"error": ["pass_threshold must be between 0.0 and 1.0"]} + return _terminal_failure_result( + ["pass_threshold must be between 0.0 and 1.0"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if grading_mode not in {"default", "default_plus_custom", "custom_only"}: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid grading mode")) - return {"error": ["grading.mode must be default, default_plus_custom, or custom_only"]} + return _terminal_failure_result( + ["grading.mode must be default, default_plus_custom, or custom_only"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if workspace_mode not in {"isolated", "group"}: reporter.emit(ProgressEvent(stage="configuration", state="failed", detail="invalid workspace mode")) - return {"error": ["skill_workspace.mode must be isolated or group"]} + return _terminal_failure_result( + ["skill_workspace.mode must be isolated or group"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) reporter.emit(ProgressEvent(stage="configuration", state="ready", detail="evaluation config validated")) reporter.emit(ProgressEvent(stage="model-resolution", state="running")) @@ -1934,7 +1982,7 @@ def _run_harbor_eval_impl( prereq_errors = _check_prerequisites(env_mode=env_mode, agents=agents) if prereq_errors: reporter.emit(ProgressEvent(stage="environment-preflight", state="failed", detail="; ".join(prereq_errors))) - return {"error": prereq_errors} + return _terminal_failure_result(prereq_errors, error_code=EvaluatorErrorCode.UNKNOWN) reporter.emit(ProgressEvent(stage="environment-preflight", state="complete", detail=env_mode)) # Resolve the effective source before constructing credential-probe targets. @@ -1946,18 +1994,27 @@ def _run_harbor_eval_impl( task_source = "evals_json" if evals_exists else "native_harbor" if native_exists else "" if task_source == "evals_json" and not evals_exists: reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail="evaluation dataset missing")) - return {"error": ["No evals/evals.json found. Run create-eval-dataset or add a dataset."]} + return _terminal_failure_result( + ["No evals/evals.json found. Run create-eval-dataset or add a dataset."], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if task_source == "native_harbor" and not native_exists: reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail="native Harbor tasks missing")) - return {"error": ["No native Harbor task source found at evals/harbor."]} + return _terminal_failure_result( + ["No native Harbor task source found at evals/harbor."], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) if task_source not in {"evals_json", "native_harbor"}: reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail="invalid task source")) - return {"error": ["harbor.task_source must be auto, evals_json, or native_harbor"]} + return _terminal_failure_result( + ["harbor.task_source must be auto, evals_json, or native_harbor"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) reporter.emit(ProgressEvent(stage="credential-validation", state="running")) if runtime_errors: reporter.emit(ProgressEvent(stage="credential-validation", state="failed", detail="; ".join(runtime_errors))) - return {"error": runtime_errors} + return _terminal_failure_result(runtime_errors, error_code=EvaluatorErrorCode.INVALID_CONFIGURATION) try: runtime_plans = _resolve_agent_runtime_plan( provider=provider, @@ -1969,7 +2026,7 @@ def _run_harbor_eval_impl( ) except ValueError as exc: reporter.emit(ProgressEvent(stage="credential-validation", state="failed", detail=str(exc))) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.INVALID_CONFIGURATION) nvidia_build_agent_import_paths = { agent: import_path for agent in agents @@ -2062,6 +2119,7 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None: } probe_errors: list[str] = [] + probe_error_codes: list[str] = [] for route_key, (selected_provider, selected_labels) in probe_targets.items(): label = ", ".join(selected_labels) try: @@ -2084,6 +2142,12 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None: safe_detail = redact_progress_detail(probe.detail, secret_values=runtime_secret_values) disposition = credential_probe_disposition(selected_provider, probe) + probe_error_code = getattr(probe, "error_code", None) + if not probe.ok and not is_registered_error_code(probe_error_code): + probe_error_code = provider_failure_error_code( + getattr(probe, "failure_kind", None), + getattr(probe, "http_status", None), + ).value if probe.ok and disposition == CredentialProbeDisposition.DEGRADED: safe_detail = "model catalog access does not verify runtime credentials for this endpoint" credential_validation_targets.append( @@ -2099,6 +2163,7 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None: judge_config["catalog_verification"] = disposition.value if disposition == CredentialProbeDisposition.FATAL: probe_errors.append(f"{label} provider verification failed: {safe_detail}") + probe_error_codes.append(validate_error_code(probe_error_code)) elif disposition == CredentialProbeDisposition.DEGRADED: probe_degraded.append(f"{label}: {safe_detail}") @@ -2110,7 +2175,10 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None: detail="; ".join(probe_errors), ) ) - return {"error": probe_errors} + return _terminal_failure_result( + probe_errors, + error_code=primary_error_code(probe_error_codes), + ) if probe_degraded: reporter.emit( ProgressEvent( @@ -2159,12 +2227,15 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None: include_values = [*workspace_config.get("include", []), *(include_skills or [])] if include_values and workspace_mode != "group": reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail="invalid included skills")) - return {"error": ["include_skills requires skill_workspace.mode=group"]} + return _terminal_failure_result( + ["include_skills requires skill_workspace.mode=group"], + error_code=EvaluatorErrorCode.INVALID_CONFIGURATION, + ) try: workspace_skills = _workspace_skills(skill_path.resolve(), include_values if workspace_mode == "group" else []) except ValueError as exc: reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail=str(exc))) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.INVALID_CONFIGURATION) root = Path(output_dir) if output_dir is not None else skill_path / "evals" / "results" try: @@ -2182,13 +2253,13 @@ def add_probe_target(label: str, selected_provider: ProviderConfig) -> None: ) except ValueError as exc: reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail=str(exc))) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.INVALID_CONFIGURATION) timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") try: run_dir = _reserve_run_dir(root, timestamp) except (OSError, RuntimeError, ValueError) as exc: reporter.emit(ProgressEvent(stage="with-skill-tasks", state="failed", detail=str(exc))) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.UNKNOWN) run_id = run_dir.name jobs_dir = run_dir / "_harbor-jobs" tasks_dir = run_dir / "_harbor-tasks" @@ -2211,22 +2282,26 @@ def _emit_run_finished(state: str, detail: str, *, include_artifacts: bool = Tru ) ) - def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]: + def _persist_pre_execution_failure( + errors: list[str], + *, + error_code: EvaluatorErrorCode | str, + ) -> dict[str, Any]: """Retain redacted probe provenance for failures after run reservation.""" - failed_result: dict[str, Any] = { - "skill_name": skill_path.name, - "execution_status": "failed", - "execution_errors": errors, - "error": errors, - "run_id": run_id, - "run_dir": str(run_dir), - "harbor_jobs_dir": str(jobs_dir), - "harbor_jobs_retained": jobs_dir.is_dir(), - "duration_seconds": round(time.monotonic() - started_at, 3), - "result_path": str(result_path), - "agents": {}, - "run_config": run_config, - } + failed_result = _terminal_failure_result(errors, error_code=error_code) + failed_result.update( + { + "skill_name": skill_path.name, + "run_id": run_id, + "run_dir": str(run_dir), + "harbor_jobs_dir": str(jobs_dir), + "harbor_jobs_retained": jobs_dir.is_dir(), + "duration_seconds": round(time.monotonic() - started_at, 3), + "result_path": str(result_path), + "agents": {}, + "run_config": run_config, + } + ) write_output_file_atomically( run_dir / "run_config.json", json.dumps(run_config, indent=2).encode("utf-8"), @@ -2244,7 +2319,7 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]: if reservation_identity is not None: remove_generated_output_root_if_owned(run_dir, expected_identity=reservation_identity) _emit_run_finished("failed", "Harbor jobs directory could not be created", include_artifacts=False) - return {"error": [str(exc)]} + return _terminal_failure_result([str(exc)], error_code=EvaluatorErrorCode.UNKNOWN) emitter = stage_native_harbor_tasks if task_source == "native_harbor" else generate_harbor_tasks resource_config = harbor_config.get("resources", {}) @@ -2359,7 +2434,10 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]: reporter.emit(ProgressEvent(stage="baseline-tasks", state="skipped", detail="baseline disabled")) except (OSError, ValueError) as exc: reporter.emit(ProgressEvent(stage=staging_failure_stage, state="failed", detail=str(exc))) - return _persist_pre_execution_failure([str(exc)]) + return _persist_pre_execution_failure( + [str(exc)], + error_code=EvaluatorErrorCode.UNKNOWN, + ) task_names = expected_task_names or [] expected_trials = len(task_names) * n_attempts @@ -2408,6 +2486,7 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]: reporter.emit(ProgressEvent(stage="agent-runtime-preflight", state="running")) preflight_errors: list[str] = [] + preflight_error_codes: list[str] = [] for agent in agents: preflight = run_agent_runtime_preflight( dataset=agent_task_dirs[agent][0], @@ -2424,10 +2503,14 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]: ) if not preflight.ok: preflight_errors.append(f"{agent} runtime preflight failed: {preflight.detail}") + preflight_error_codes.append(preflight.error_code or EvaluatorErrorCode.UNKNOWN.value) if preflight_errors: detail = "; ".join(preflight_errors) reporter.emit(ProgressEvent(stage="agent-runtime-preflight", state="failed", detail=detail)) - failed_result = _persist_pre_execution_failure(preflight_errors) + failed_result = _persist_pre_execution_failure( + preflight_errors, + error_code=primary_error_code(preflight_error_codes), + ) _emit_run_finished("failed", "agent runtime preflight failed") return failed_result reporter.emit( @@ -2582,6 +2665,8 @@ def _emit_started_agents() -> None: results["execution_status"] = "failed" results["execution_errors"] = execution_errors results["error"] = execution_errors + if results.get("execution_status") == "failed": + results.setdefault("error_code", EvaluatorErrorCode.UNKNOWN.value) reporter.emit(ProgressEvent(stage="report", state="running")) try: (run_dir / "run_config.json").write_text(json.dumps(run_config, indent=2), encoding="utf-8") diff --git a/src/skillevaluator/tier3/harbor/runtime_preflight.py b/src/skillevaluator/tier3/harbor/runtime_preflight.py index d7ce0d55..34616ff0 100644 --- a/src/skillevaluator/tier3/harbor/runtime_preflight.py +++ b/src/skillevaluator/tier3/harbor/runtime_preflight.py @@ -13,7 +13,7 @@ import ssl import subprocess from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import InitVar, dataclass from enum import StrEnum from pathlib import Path from queue import Empty, Queue @@ -33,6 +33,7 @@ ClientError, ConfigNotFound, ConfigParseError, + ConnectTimeoutError, CredentialRetrievalError, DataNotFoundError, EndpointProviderError, @@ -52,6 +53,7 @@ ParamValidationError, PartialCredentialsError, ProfileNotFound, + ReadTimeoutError, RefreshWithMFAUnsupportedError, ServiceNotInRegionError, SSOTokenLoadError, @@ -65,6 +67,7 @@ SSLError as BotocoreSSLError, ) +from skillevaluator.error_codes import EvaluatorErrorCode, provider_failure_error_code, validate_error_code from skillevaluator.model_catalog import ( ModelCatalogError, ModelCatalogFailureKind, @@ -89,6 +92,15 @@ class PreflightResult: model: str detail: str job_name: str + error_code: str | None = None + + def __post_init__(self) -> None: + if self.ok: + if self.error_code is not None: + raise ValueError("successful runtime preflight cannot carry an error code") + return + resolved = self.error_code or EvaluatorErrorCode.UNKNOWN + object.__setattr__(self, "error_code", validate_error_code(resolved)) @dataclass(frozen=True) @@ -102,6 +114,23 @@ class ModelProbeResult: failure_kind: ModelCatalogFailureKind | None = None http_status: int | None = None catalog_authoritative: bool = True + error_code: str | None = None + timed_out: InitVar[bool] = False + + def __post_init__(self, timed_out: bool) -> None: + if self.ok: + if self.error_code is not None or timed_out: + raise ValueError("successful model probe cannot carry failure metadata") + return + expected = provider_failure_error_code( + self.failure_kind, + self.http_status, + timed_out=timed_out, + ).value + resolved = expected if self.error_code is None else validate_error_code(self.error_code) + if self.failure_kind is not None and resolved != expected: + raise ValueError("model probe error code contradicts structured failure metadata") + object.__setattr__(self, "error_code", resolved) class CredentialProbeDisposition(StrEnum): @@ -1034,6 +1063,7 @@ def _bedrock_exception_result( ) -> ModelProbeResult: """Convert a Bedrock SDK failure to a redacted, policy-safe result.""" http_status = None + timed_out = False if isinstance(exc, ClientError): failure_kind, http_status = _bedrock_client_error_kind(exc) elif ( @@ -1061,7 +1091,8 @@ def _bedrock_exception_result( and exc.errno in _BEDROCK_CREDENTIAL_PROCESS_TRANSIENT_ERRNOS and _bedrock_credential_process(session) is not None ): - failure_kind = ModelCatalogFailureKind.UNAVAILABLE + failure_kind = ModelCatalogFailureKind.LOCAL_PROCESS + timed_out = exc.errno == errno.ETIMEDOUT elif _is_bedrock_local_cached_token_error(exc): failure_kind = ModelCatalogFailureKind.AUTHENTICATION elif isinstance(exc, ValueError): @@ -1077,6 +1108,9 @@ def _bedrock_exception_result( failure_kind = ModelCatalogFailureKind.UNKNOWN elif isinstance(exc, BotocoreSSLError) and _is_invalid_bedrock_ca_bundle(session): failure_kind = ModelCatalogFailureKind.INVALID_CONFIGURATION + elif isinstance(exc, (ConnectTimeoutError, ReadTimeoutError)): + failure_kind = ModelCatalogFailureKind.UNAVAILABLE + timed_out = True elif isinstance(exc, BotoCoreError): failure_kind = ModelCatalogFailureKind.UNAVAILABLE else: @@ -1088,6 +1122,7 @@ def _bedrock_exception_result( f"Bedrock model catalog request failed: {type(exc).__name__}", failure_kind=failure_kind, http_status=http_status, + timed_out=timed_out, ) @@ -1175,6 +1210,7 @@ def _probe_bedrock_model(provider: ProviderConfig, *, timeout_seconds: float) -> provider.model, f"model {provider.model} is not listed", catalog_authoritative=catalog_authoritative, + error_code=EvaluatorErrorCode.MODEL_NOT_FOUND, ) return ModelProbeResult( True, @@ -1214,6 +1250,8 @@ def _probe_bedrock_model_with_deadline( provider.model, "Bedrock model catalog request timed out", failure_kind=ModelCatalogFailureKind.UNAVAILABLE, + error_code=EvaluatorErrorCode.DEPENDENCY_TIMEOUT, + timed_out=True, ) def run_probe() -> None: @@ -1241,6 +1279,8 @@ def run_probe() -> None: provider.model, "Bedrock model catalog request timed out", failure_kind=ModelCatalogFailureKind.UNAVAILABLE, + error_code=EvaluatorErrorCode.DEPENDENCY_TIMEOUT, + timed_out=True, ) try: @@ -1266,6 +1306,8 @@ def run_probe() -> None: provider.model, "Bedrock model catalog request timed out", failure_kind=ModelCatalogFailureKind.UNAVAILABLE, + error_code=EvaluatorErrorCode.DEPENDENCY_TIMEOUT, + timed_out=True, ) @@ -1298,6 +1340,8 @@ def probe_model(provider: ProviderConfig, *, timeout_seconds: float = 15.0) -> M str(exc), failure_kind=exc.kind, http_status=exc.http_status, + error_code=exc.error_code, + timed_out=exc.error_code == EvaluatorErrorCode.DEPENDENCY_TIMEOUT, ) available = {record.id for record in records} if provider.model not in available: @@ -1310,6 +1354,8 @@ def probe_model(provider: ProviderConfig, *, timeout_seconds: float = 15.0) -> M provider.model, "model catalog request timed out", failure_kind=ModelCatalogFailureKind.UNAVAILABLE, + error_code=EvaluatorErrorCode.DEPENDENCY_TIMEOUT, + timed_out=True, ) try: resolved = fetch_anthropic_model_record( @@ -1326,6 +1372,7 @@ def probe_model(provider: ProviderConfig, *, timeout_seconds: float = 15.0) -> M f"model {provider.model} is not available", failure_kind=ModelCatalogFailureKind.MODEL_NOT_FOUND, http_status=404, + error_code=EvaluatorErrorCode.MODEL_NOT_FOUND, ) return ModelProbeResult( False, @@ -1334,6 +1381,8 @@ def probe_model(provider: ProviderConfig, *, timeout_seconds: float = 15.0) -> M str(exc), failure_kind=exc.kind, http_status=exc.http_status, + error_code=exc.error_code, + timed_out=exc.error_code == EvaluatorErrorCode.DEPENDENCY_TIMEOUT, ) return ModelProbeResult( True, @@ -1341,7 +1390,13 @@ def probe_model(provider: ProviderConfig, *, timeout_seconds: float = 15.0) -> M provider.model, f"model {provider.model} resolves to {resolved.id}", ) - return ModelProbeResult(False, provider.provider, provider.model, f"model {provider.model} is not listed") + return ModelProbeResult( + False, + provider.provider, + provider.model, + f"model {provider.model} is not listed", + error_code=EvaluatorErrorCode.MODEL_NOT_FOUND, + ) return ModelProbeResult(True, provider.provider, provider.model, f"model {provider.model} is available") @@ -1364,7 +1419,14 @@ def run_agent_runtime_preflight( task_name = _first_task_name(dataset) job_name = f"runtime-preflight-{agent}" if task_name is None: - return PreflightResult(False, agent, model, "No staged tasks are available for runtime preflight.", job_name) + return PreflightResult( + False, + agent, + model, + "No staged tasks are available for runtime preflight.", + job_name, + EvaluatorErrorCode.INVALID_CONFIGURATION, + ) command = build_harbor_run_command( dataset_path=dataset, @@ -1401,17 +1463,32 @@ def run_agent_runtime_preflight( model, f"Agent runtime preflight timed out after {timeout_seconds}s.", job_name, + EvaluatorErrorCode.EXECUTION_TIMEOUT, ) except OSError as exc: - return PreflightResult(False, agent, model, f"Agent runtime preflight could not start: {exc}", job_name) + return PreflightResult( + False, + agent, + model, + f"Agent runtime preflight could not start: {exc}", + job_name, + EvaluatorErrorCode.PROCESS_SPAWN_FAILED, + ) if completed.returncode != 0: output = "\n".join(part for part in (completed.stderr, completed.stdout) if part).strip() detail = _redact_detail(output, run_env) or f"harbor run exited {completed.returncode}" - return PreflightResult(False, agent, model, detail, job_name) + return PreflightResult(False, agent, model, detail, job_name, EvaluatorErrorCode.PROCESS_EXITED) ok, detail = validate_harbor_agent_only_job_result( jobs_dir / job_name / "result.json", expected_trials=1, ) - return PreflightResult(ok, agent, model, _redact_detail(detail, run_env), job_name) + return PreflightResult( + ok, + agent, + model, + _redact_detail(detail, run_env), + job_name, + None if ok else EvaluatorErrorCode.JOB_RESULT_INVALID, + ) diff --git a/src/skillevaluator/tier3/results_location.py b/src/skillevaluator/tier3/results_location.py index 09c30fb5..e9a799ea 100644 --- a/src/skillevaluator/tier3/results_location.py +++ b/src/skillevaluator/tier3/results_location.py @@ -16,6 +16,7 @@ from pathlib import Path from uuid import uuid4 +from skillevaluator.error_codes import is_registered_error_code from skillevaluator.tier3.output_provenance import GENERATED_OUTPUT_MARKER, is_generated_output_root from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot @@ -366,6 +367,7 @@ def _current_result_identity_is_valid(candidate: Path, run_config: dict[object, ): return False attempt_policy = result.get("attempt_policy") + error_code = result.get("error_code") if ( not isinstance(result.get("skill_name"), str) or not result["skill_name"] @@ -379,6 +381,8 @@ def _current_result_identity_is_valid(candidate: Path, run_config: dict[object, or not isinstance(attempt_policy.get("stop_on_pass"), bool) or not isinstance(attempt_policy.get("score_definition"), str) or not attempt_policy["score_definition"] + or (error_code is not None and result.get("execution_status") != "failed") + or (error_code is not None and not is_registered_error_code(error_code)) ): return False return _recorded_path_matches(result.get("run_dir"), candidate) and _recorded_path_matches( diff --git a/tests/test_error_codes.py b/tests/test_error_codes.py new file mode 100644 index 00000000..a36ad438 --- /dev/null +++ b/tests/test_error_codes.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Golden contract tests for stable evaluator error codes.""" + +from __future__ import annotations + +import json +from types import MappingProxyType + +import pytest + +from skillevaluator.error_codes import ( + ERROR_CODE_PATTERN_TEXT, + ERROR_CODE_REGISTRY, + ErrorDomain, + EvaluatorErrorCode, + error_code_schema, + is_registered_error_code, + primary_error_code, + provider_failure_error_code, + validate_error_code, +) + + +def test_error_code_registry_matches_append_only_public_golden() -> None: + golden = [ + ("SKILLEVALUATOR-AUTH-002", "AUTH", "Provider authentication failed."), + ("SKILLEVALUATOR-AUTH-003", "AUTH", "Provider authorization failed."), + ("SKILLEVALUATOR-CONFIG-001", "CONFIG", "Evaluator configuration was invalid."), + ("SKILLEVALUATOR-DEPENDENCY-001", "DEPENDENCY", "A required dependency was unavailable."), + ("SKILLEVALUATOR-DEPENDENCY-003", "DEPENDENCY", "The configured model was not found."), + ("SKILLEVALUATOR-DEPENDENCY-005", "DEPENDENCY", "A required dependency timed out."), + ( + "SKILLEVALUATOR-DEPENDENCY-006", + "DEPENDENCY", + "A required dependency rate-limited the evaluator.", + ), + ( + "SKILLEVALUATOR-DEPENDENCY-007", + "DEPENDENCY", + "The selected dependency operation is unsupported.", + ), + ( + "SKILLEVALUATOR-DEPENDENCY-008", + "DEPENDENCY", + "A required dependency returned an invalid response.", + ), + ( + "SKILLEVALUATOR-DEPENDENCY-009", + "DEPENDENCY", + "A required dependency returned another HTTP failure.", + ), + ("SKILLEVALUATOR-UNKNOWN-001", "UNKNOWN", "The evaluator could not classify the failure."), + ( + "SKILLEVALUATOR-RUNTIME-005", + "RUNTIME", + "The evaluator could not start a runtime process.", + ), + ("SKILLEVALUATOR-RUNTIME-007", "RUNTIME", "Evaluator runtime execution timed out."), + ( + "SKILLEVALUATOR-RUNTIME-008", + "RUNTIME", + "An evaluator runtime process exited unsuccessfully.", + ), + ( + "SKILLEVALUATOR-CONTRACT-007", + "CONTRACT", + "The evaluator runtime produced an invalid job result.", + ), + ] + + assert [ + (code, definition.domain.value, definition.summary) for code, definition in ERROR_CODE_REGISTRY.items() + ] == golden + + +def test_error_code_registry_is_immutable_and_schema_is_serializable() -> None: + assert isinstance(ERROR_CODE_REGISTRY, MappingProxyType) + assert ERROR_CODE_PATTERN_TEXT == r"^SKILLEVALUATOR-[A-Z][A-Z0-9]*-[0-9]{3}$" + schema = error_code_schema() + assert schema["pattern"] == ERROR_CODE_PATTERN_TEXT + assert schema["enum"] == list(ERROR_CODE_REGISTRY) + assert json.loads(json.dumps(schema)) == schema + schema["enum"] = [] + assert error_code_schema()["enum"] == list(ERROR_CODE_REGISTRY) + assert {definition.domain for definition in ERROR_CODE_REGISTRY.values()} == set(ErrorDomain) + with pytest.raises(TypeError): + ERROR_CODE_REGISTRY["SKILLEVALUATOR-UNKNOWN-999"] = ERROR_CODE_REGISTRY[ # type: ignore[index] + EvaluatorErrorCode.UNKNOWN.value + ] + + +@pytest.mark.parametrize( + ("failure_kind", "http_status", "timed_out", "expected"), + [ + ("authentication", 401, False, "SKILLEVALUATOR-AUTH-002"), + ("authorization", 403, False, "SKILLEVALUATOR-AUTH-003"), + ("invalid_configuration", None, False, "SKILLEVALUATOR-CONFIG-001"), + ("model_not_found", 404, False, "SKILLEVALUATOR-DEPENDENCY-003"), + ("unsupported", 405, False, "SKILLEVALUATOR-DEPENDENCY-007"), + ("invalid_response", None, False, "SKILLEVALUATOR-DEPENDENCY-008"), + ("other_http", 418, False, "SKILLEVALUATOR-DEPENDENCY-009"), + ("unknown", None, False, "SKILLEVALUATOR-UNKNOWN-001"), + ("unavailable", 429, False, "SKILLEVALUATOR-DEPENDENCY-006"), + ("unavailable", 408, False, "SKILLEVALUATOR-DEPENDENCY-005"), + ("unavailable", None, True, "SKILLEVALUATOR-DEPENDENCY-005"), + ("unavailable", 503, False, "SKILLEVALUATOR-DEPENDENCY-001"), + ("unavailable", None, False, "SKILLEVALUATOR-DEPENDENCY-001"), + ("local_process", None, False, "SKILLEVALUATOR-RUNTIME-005"), + ("local_process", None, True, "SKILLEVALUATOR-RUNTIME-007"), + ], +) +def test_provider_failure_mapping_is_structured_and_stable( + failure_kind: str, + http_status: int | None, + timed_out: bool, + expected: str, +) -> None: + assert provider_failure_error_code(failure_kind, http_status, timed_out=timed_out).value == expected + + +@pytest.mark.parametrize( + "value", + [ + "skillevaluator-auth-002", + "SKILLEVALUATOR-AUTH-2", + "SKILLEVALUATOR-SERVICE-999", + "SKILLEVALUATOR-AUTH-002\n", + "SKILL\u212aEVALUATOR-AUTH-002", + None, + ], +) +def test_error_code_validation_rejects_noncanonical_values(value: object) -> None: + assert is_registered_error_code(value) is False + with pytest.raises(ValueError, match="unregistered"): + validate_error_code(value) + + +def test_primary_error_code_requires_consensus_and_falls_back_to_unknown() -> None: + assert primary_error_code(["invalid", EvaluatorErrorCode.AUTHORIZATION]) == "SKILLEVALUATOR-AUTH-003" + assert primary_error_code([]) == "SKILLEVALUATOR-UNKNOWN-001" + assert ( + primary_error_code([EvaluatorErrorCode.AUTHENTICATION, EvaluatorErrorCode.INVALID_CONFIGURATION]) + == "SKILLEVALUATOR-UNKNOWN-001" + ) diff --git a/tests/test_harbor_local_mode.py b/tests/test_harbor_local_mode.py index cdda2329..da0121d7 100644 --- a/tests/test_harbor_local_mode.py +++ b/tests/test_harbor_local_mode.py @@ -3197,12 +3197,16 @@ def test_run_harbor_eval_rejects_native_windows_before_provider_or_config( result = runner.run_harbor_eval(tmp_path, ["opencode"], env_mode="local") + expected_errors = [ + "Native Windows local mode is unsupported, including with " + "SKILLEVALUATOR_LOCAL_SANDBOX=prefer or off. " + "Use WSL2 for Linux local mode or --env-mode docker." + ] assert result == { - "error": [ - "Native Windows local mode is unsupported, including with " - "SKILLEVALUATOR_LOCAL_SANDBOX=prefer or off. " - "Use WSL2 for Linux local mode or --env-mode docker." - ] + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-UNKNOWN-001", + "execution_errors": expected_errors, + "error": expected_errors, } diff --git a/tests/test_harbor_runtime_preflight.py b/tests/test_harbor_runtime_preflight.py index 634dc5c8..6ae0079c 100644 --- a/tests/test_harbor_runtime_preflight.py +++ b/tests/test_harbor_runtime_preflight.py @@ -65,6 +65,30 @@ from skillevaluator.tier3.harbor.collector import validate_harbor_job_result +def test_model_probe_rejects_contradictory_structured_error_code() -> None: + with pytest.raises(ValueError, match="contradicts"): + runtime_preflight.ModelProbeResult( + False, + "openai", + "requested-model", + "safe detail", + failure_kind="authentication", + http_status=401, + error_code="SKILLEVALUATOR-CONFIG-001", + ) + + timeout = runtime_preflight.ModelProbeResult( + False, + "openai", + "requested-model", + "safe detail", + failure_kind="unavailable", + error_code="SKILLEVALUATOR-DEPENDENCY-005", + timed_out=True, + ) + assert timeout.error_code == "SKILLEVALUATOR-DEPENDENCY-005" + + def _dataset(tmp_path: Path) -> Path: dataset = tmp_path / "tasks" (dataset / "case-002").mkdir(parents=True) @@ -521,6 +545,8 @@ def test_runtime_preflight_redacts_and_sanitizes_retained_trial_exception( assert result.ok is False assert "NonZeroAgentExitCodeError" in result.detail + assert result.error_code == "SKILLEVALUATOR-CONTRACT-007" + assert "DEPENDENCY" not in result.error_code assert secret not in result.detail assert "\x1b" not in result.detail assert len(result.detail) <= 2000 @@ -601,6 +627,8 @@ def test_runtime_preflight_reports_agent_start_failure(monkeypatch, tmp_path: Pa assert result.ok is False assert result.agent == "opencode" + assert result.error_code == "SKILLEVALUATOR-RUNTIME-008" + assert "DEPENDENCY" not in result.error_code assert "401 Unauthorized" in result.detail @@ -626,10 +654,33 @@ def timeout(*_args, **kwargs): ) assert result.ok is False + assert result.error_code == "SKILLEVALUATOR-RUNTIME-007" + assert "DEPENDENCY" not in result.error_code assert "timed out after 30s" in result.detail assert secret not in result.detail +def test_runtime_preflight_process_spawn_failure_is_local_runtime_error(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr(runtime_preflight, "build_harbor_run_command", lambda **_kwargs: ["harbor", "run"]) + monkeypatch.setattr( + runtime_preflight.subprocess, + "run", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("runtime executable unavailable")), + ) + + result = runtime_preflight.run_agent_runtime_preflight( + dataset=_dataset(tmp_path), + agent="opencode", + model="model", + env_mode="local", + jobs_dir=tmp_path / "jobs", + run_env={}, + ) + + assert result.error_code == "SKILLEVALUATOR-RUNTIME-005" + assert "DEPENDENCY" not in result.error_code + + def test_runtime_preflight_rejects_empty_task_tree(tmp_path: Path) -> None: dataset = tmp_path / "tasks" dataset.mkdir() @@ -644,6 +695,7 @@ def test_runtime_preflight_rejects_empty_task_tree(tmp_path: Path) -> None: ) assert result.ok is False + assert result.error_code == "SKILLEVALUATOR-CONFIG-001" assert "no staged tasks" in result.detail.lower() @@ -3119,9 +3171,18 @@ def test_bedrock_model_probe_classifies_structural_credential_process_output_cau assert str(credential_process) not in result.detail +@pytest.mark.parametrize( + ("error_number", "expected_error_code"), + [ + (errno.EAGAIN, "SKILLEVALUATOR-RUNTIME-005"), + (errno.ETIMEDOUT, "SKILLEVALUATOR-RUNTIME-007"), + ], +) def test_bedrock_model_probe_degrades_transient_credential_process_spawn_error( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + error_number: int, + expected_error_code: str, ) -> None: original_init = botocore_credentials.ProcessProvider.__init__ @@ -3129,7 +3190,7 @@ def failing_init(self, *args, **kwargs) -> None: original_init(self, *args, **kwargs) def fail_to_spawn(*_args, **_kwargs): - raise BlockingIOError(errno.EAGAIN, "private temporary spawn failure") + raise OSError(error_number, "private temporary spawn failure") self._popen = fail_to_spawn @@ -3154,7 +3215,9 @@ def fail_to_spawn(*_args, **_kwargs): result = runtime_preflight.probe_model(provider, timeout_seconds=5) - assert result.failure_kind == "unavailable" + assert result.failure_kind == "local_process" + assert result.error_code == expected_error_code + assert "DEPENDENCY" not in result.error_code assert runtime_preflight.credential_probe_disposition(provider, result) == "degraded" assert "private" not in result.detail @@ -3586,6 +3649,7 @@ def fail_preflight(**kwargs): ) assert result["execution_status"] == "failed" + assert result["error_code"] == "SKILLEVALUATOR-UNKNOWN-001" assert result["execution_errors"] == ["opencode runtime preflight failed: 401 Unauthorized"] assert preflight_run_env["LLM_JUDGE_MODEL"] == "host-legacy" assert preflight_run_env["SKILL_EVAL_JUDGE_MODEL"] == "host-legacy" @@ -3604,6 +3668,7 @@ def fail_preflight(**kwargs): "model catalog access does not verify runtime credentials for this endpoint" } persisted = json.loads(result_path.read_text(encoding="utf-8")) + assert persisted["error_code"] == "SKILLEVALUATOR-UNKNOWN-001" assert persisted["run_config"] == result["run_config"] run_config_path = Path(result["run_dir"]) / "run_config.json" assert json.loads(run_config_path.read_text(encoding="utf-8")) == result["run_config"] diff --git a/tests/test_model_catalog.py b/tests/test_model_catalog.py index 22ebc4b1..4f41d1c5 100644 --- a/tests/test_model_catalog.py +++ b/tests/test_model_catalog.py @@ -18,6 +18,7 @@ from skillevaluator import model_catalog from skillevaluator.model_catalog import ( ModelCatalogError, + ModelCatalogFailureKind, ModelRecord, fetch_anthropic_model_record, fetch_model_records, @@ -531,9 +532,28 @@ def test_fetch_never_exposes_http_body_reason_url_or_key(monkeypatch) -> None: message = str(caught.value) assert message == "model catalog returned HTTP 401" + assert caught.value.error_code == "SKILLEVALUATOR-AUTH-002" assert all(secret not in message for secret in ("top-secret-key", "secret-path", "query-secret", "password")) +def test_catalog_error_rejects_code_that_contradicts_structured_metadata() -> None: + with pytest.raises(ValueError, match="contradicts"): + ModelCatalogError( + "safe detail", + kind=ModelCatalogFailureKind.AUTHENTICATION, + http_status=401, + error_code="SKILLEVALUATOR-CONFIG-001", + ) + + timeout = ModelCatalogError( + "safe detail", + kind=ModelCatalogFailureKind.UNAVAILABLE, + error_code="SKILLEVALUATOR-DEPENDENCY-005", + timed_out=True, + ) + assert timeout.error_code == "SKILLEVALUATOR-DEPENDENCY-005" + + @pytest.mark.parametrize( ("status", "expected_kind"), [ @@ -566,6 +586,19 @@ def test_fetch_classifies_http_failures_without_exposing_response( assert getattr(caught.value, "kind", None) == expected_kind assert getattr(caught.value, "http_status", None) == status + assert ( + caught.value.error_code + == { + 401: "SKILLEVALUATOR-AUTH-002", + 403: "SKILLEVALUATOR-AUTH-003", + 404: "SKILLEVALUATOR-DEPENDENCY-007", + 405: "SKILLEVALUATOR-DEPENDENCY-007", + 408: "SKILLEVALUATOR-DEPENDENCY-005", + 429: "SKILLEVALUATOR-DEPENDENCY-006", + 500: "SKILLEVALUATOR-DEPENDENCY-001", + 418: "SKILLEVALUATOR-DEPENDENCY-009", + }[status] + ) assert "top-secret-key" not in str(caught.value) diff --git a/tests/test_results_location.py b/tests/test_results_location.py index 758b6287..963176ee 100644 --- a/tests/test_results_location.py +++ b/tests/test_results_location.py @@ -246,8 +246,19 @@ def test_latest_results_rejects_authenticated_current_run_with_malformed_config_ lambda result: result["agents"]["opencode"].update({"with_skill": []}), lambda result: result["agents"]["opencode"]["model_resolution"].update({"model": "other-model"}), lambda result: result.pop("execution_status"), + lambda result: result.update({"error_code": "SKILLEVALUATOR-SERVICE-999"}), + lambda result: result.update({"error_code": "SKILLEVALUATOR-AUTH-002"}), ], - ids=("empty", "extra-agent", "non-object-agent", "non-object-scores", "model-mismatch", "missing-status"), + ids=( + "empty", + "extra-agent", + "non-object-agent", + "non-object-scores", + "model-mismatch", + "missing-status", + "unregistered-error-code", + "success-with-error-code", + ), ) def test_latest_results_rejects_authenticated_current_run_with_malformed_result_schema( tmp_path: Path, diff --git a/tests/test_tier3_progress.py b/tests/test_tier3_progress.py index d3df5bfb..14ff788c 100644 --- a/tests/test_tier3_progress.py +++ b/tests/test_tier3_progress.py @@ -843,6 +843,9 @@ def emit_tasks(_skill, output: Path, **_kwargs): assert probe_calls and probe_calls[0].provider == "openai" assert result.get("error") + assert result["execution_status"] == "failed" + assert result["error_code"] == "SKILLEVALUATOR-AUTH-002" + assert result["execution_errors"] == result["error"] assert "HTTP 401" in json.dumps(result) assert image_calls == [] assert task_calls == [] @@ -853,6 +856,46 @@ def emit_tasks(_skill, output: Path, **_kwargs): assert secret not in rendered +def test_conflicting_terminal_probe_codes_collapse_to_unknown( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-runtime-secret") + + def reject_route(selected_provider): + if selected_provider.provider == "anthropic": + return SimpleNamespace( + ok=False, + provider=selected_provider.provider, + model=selected_provider.model, + detail="model catalog configuration is invalid", + failure_kind="invalid_configuration", + http_status=None, + ) + return SimpleNamespace( + ok=False, + provider=selected_provider.provider, + model=selected_provider.model, + detail="model catalog returned HTTP 401", + failure_kind="authentication", + http_status=401, + ) + + runner, skill = _stub_runner(monkeypatch, tmp_path, model_probe=reject_route) + + result = runner.run_harbor_eval( + skill, + ["codex", "claude-code"], + agent_models={"claude-code": "anthropic/claude-test"}, + output_dir=tmp_path / "results", + agent_runtime_preflight=False, + ) + + assert result["execution_status"] == "failed" + assert result["error_code"] == "SKILLEVALUATOR-UNKNOWN-001" + assert len(result["execution_errors"]) == 2 + + @pytest.mark.parametrize( ("provider_name", "provider_base_url", "failure_kind", "http_status", "detail"), [ @@ -1066,7 +1109,12 @@ def record_probe(selected_provider): progress_reporter=reporter, ) - assert result == {"error": ["No native Harbor task source found at evals/harbor."]} + assert result == { + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-CONFIG-001", + "execution_errors": ["No native Harbor task source found at evals/harbor."], + "error": ["No native Harbor task source found at evals/harbor."], + } assert probe_calls == [] transitions = [(event.stage, event.state) for event in reporter.events] assert ("environment-preflight", "complete") in transitions @@ -1737,7 +1785,12 @@ def fail_jobs_mkdir(path: Path, *args: Any, **kwargs: Any) -> None: progress_reporter=reporter, ) - assert result == {"error": ["[Errno 28] No space left on device"]} + assert result == { + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-UNKNOWN-001", + "execution_errors": ["[Errno 28] No space left on device"], + "error": ["[Errno 28] No space left on device"], + } assert results_root.is_dir() assert list(results_root.iterdir()) == [] transitions = [(event.stage, event.state) for event in reporter.events] @@ -1765,7 +1818,12 @@ def fail_jobs_mkdir(path: Path, *args: Any, **kwargs: Any) -> None: result = runner.run_harbor_eval(skill, ["codex"], output_dir=results_root) - assert result == {"error": ["injected jobs directory failure"]} + assert result == { + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-UNKNOWN-001", + "execution_errors": ["injected jobs directory failure"], + "error": ["injected jobs directory failure"], + } assert list(results_root.iterdir()) == [] @@ -1790,7 +1848,12 @@ def fail_after_tampering(path: Path, *args: Any, **kwargs: Any) -> None: result = runner.run_harbor_eval(skill, ["codex"], output_dir=tmp_path / "results") - assert result == {"error": ["injected jobs directory failure"]} + assert result == { + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-UNKNOWN-001", + "execution_errors": ["injected jobs directory failure"], + "error": ["injected jobs directory failure"], + } assert len(reserved_run) == 1 assert (reserved_run[0] / "preserve.txt").read_text(encoding="utf-8") == "unowned\n" @@ -2251,7 +2314,12 @@ def test_runner_reports_known_plan_without_claiming_failed_preflight_ready( result = runner.run_harbor_eval(skill, ["codex"], progress_reporter=reporter) - assert result == {"error": ["Docker is unavailable"]} + assert result == { + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-UNKNOWN-001", + "execution_errors": ["Docker is unavailable"], + "error": ["Docker is unavailable"], + } known_plan = reporter.plans[-1] assert known_plan.provider == "openai" assert known_plan.agent_models == (("codex", "gpt-5"),) @@ -2285,7 +2353,12 @@ def test_runner_does_not_mark_invalid_configuration_ready( result = runner.run_harbor_eval(skill, ["codex"], progress_reporter=reporter) - assert result == {"error": ["n_attempts must be >= 1"]} + assert result == { + "execution_status": "failed", + "error_code": "SKILLEVALUATOR-CONFIG-001", + "execution_errors": ["n_attempts must be >= 1"], + "error": ["n_attempts must be >= 1"], + } transitions = [(event.stage, event.state) for event in reporter.events] assert transitions == [ ("configuration", "running"),