From ec44884944933f07fdc1084cc68a9e5fb89f18a6 Mon Sep 17 00:00:00 2001 From: CK0607 Date: Mon, 3 Aug 2026 11:44:33 +0530 Subject: [PATCH 1/3] feat(cli): add resource QA agent commands Expose existing Platform QA workflows for environment and taskset resources with strict result selection, waiting, and machine-readable output. Co-authored-by: Cursor --- docs/v6/reference/cli.mdx | 16 ++ hud/cli/__init__.py | 2 + hud/cli/qa.py | 331 ++++++++++++++++++++++++ hud/cli/tests/test_qa.py | 415 +++++++++++++++++++++++++++++++ hud/utils/platform.py | 2 +- hud/utils/tests/test_platform.py | 13 + 6 files changed, 778 insertions(+), 1 deletion(-) create mode 100644 hud/cli/qa.py create mode 100644 hud/cli/tests/test_qa.py diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index 27605ecf7..d1b71bafe 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -135,6 +135,22 @@ hud sync env # sync environment metadata External benchmark formats (currently Harbor) load directly into the runtime as `Taskset`s - no conversion step. See [Harbor interop](/v6/advanced/harbor-convert). +### Resource QA + +Use Platform QA agents with Environment or Taskset subjects: + +```bash +hud qa agents --subject-type environment +hud qa run +hud qa results environment +``` + +`hud qa run` reuses matching evidence by default and waits for every requested +subject. Pass `--overwrite` to create a fresh attempt, `--no-wait` to return +after launch, or `--json` for machine-readable output. Exit code `1` means the +completed QA result failed or remained unknown, `2` means the request was +rejected, and `3` means execution, polling, or the response contract failed. + ## Inspect ### `hud jobs []` diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index ed4f6e1d1..336a4ab4b 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -38,6 +38,7 @@ from .jobs import jobs_app # noqa: E402 from .login import login_command # noqa: E402 from .models import models_app # noqa: E402 +from .qa import qa_app # noqa: E402 from .serve import serve_command # noqa: E402 from .sync import sync_app # noqa: E402 from .task import task_app # noqa: E402 @@ -53,6 +54,7 @@ app.add_typer(models_app, name="models") app.add_typer(jobs_app, name="jobs") app.add_typer(trace_app, name="trace") +app.add_typer(qa_app, name="qa") @app.command(name="set") diff --git a/hud/cli/qa.py b/hud/cli/qa.py new file mode 100644 index 000000000..96d02b8d6 --- /dev/null +++ b/hud/cli/qa.py @@ -0,0 +1,331 @@ +"""Discover, run, and inspect resource-scoped platform QA agents.""" + +from __future__ import annotations + +import json +import time +from typing import Any, NoReturn, cast + +import typer + +from hud.cli.utils.api import require_api_key +from hud.utils.exceptions import HudNetworkError, HudRequestError, HudTimeoutError +from hud.utils.platform import PlatformClient + +_POLL_INTERVAL_SECONDS = 2.0 +_RESOURCE_SUBJECT_TYPES = {"environment", "taskset"} +_TERMINAL_STATUSES = {"completed", "error"} + +qa_app = typer.Typer( + name="qa", + help="Discover, run, and inspect platform QA agents.", + add_completion=False, + rich_markup_mode="rich", + no_args_is_help=True, +) + + +def _request_error(message: str) -> NoReturn: + typer.echo(message, err=True) + raise typer.Exit(2) + + +def _execution_error(message: str) -> NoReturn: + typer.echo(message, err=True) + raise typer.Exit(3) + + +def _platform() -> PlatformClient: + try: + require_api_key("use platform QA agents") + except typer.Exit as exc: + raise typer.Exit(2) from exc + return PlatformClient.from_settings() + + +def _subject_type(value: str) -> str: + normalized = value.strip().lower() + if normalized not in _RESOURCE_SUBJECT_TYPES: + choices = ", ".join(sorted(_RESOURCE_SUBJECT_TYPES)) + _request_error(f"Subject type must be one of: {choices}.") + return normalized + + +def _dict_list(value: object, *, label: str) -> list[dict[str, Any]]: + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + _execution_error(f"Platform returned invalid {label}.") + return cast("list[dict[str, Any]]", value) + + +def _print_json(value: object) -> None: + typer.echo(json.dumps(value, indent=2, sort_keys=True, default=str)) + + +def _canonical_id(value: object) -> str: + """Match UUID-like identifiers independently of accepted hex casing.""" + return str(value).casefold() + + +def _selected_trace_ids( + runs: list[dict[str, Any]], + *, + agent_id: str, + subject_ids: list[str], +) -> dict[str, str]: + selected = { + _canonical_id(run["subject_id"]): _canonical_id(run["analysis_trace_id"]) + for run in runs + if isinstance(run.get("subject_id"), str) + and isinstance(run.get("analysis_trace_id"), str) + and _canonical_id(run.get("qa_agent_id")) == _canonical_id(agent_id) + } + expected_subject_ids = {_canonical_id(subject_id) for subject_id in subject_ids} + if ( + len(runs) != len(subject_ids) + or len(selected) != len(runs) + or set(selected) != expected_subject_ids + ): + _execution_error("Platform did not select exactly one QA result per requested resource.") + return selected + + +def _result_verdict(result: dict[str, Any]) -> tuple[str, str | None]: + canonical = result.get("canonical_result") + if isinstance(canonical, dict): + canonical_dict = cast("dict[str, Any]", canonical) + verdict = canonical_dict.get("verdict") + summary = canonical_dict.get("summary") + if isinstance(verdict, str): + return verdict, summary if isinstance(summary, str) else None + status = result.get("status") + error = result.get("error") + return ( + status if isinstance(status, str) else "unknown", + error if isinstance(error, str) else None, + ) + + +def _render_results(results: list[dict[str, Any]]) -> None: + if not results: + typer.echo("No QA results found.") + return + for result in results: + verdict, summary = _result_verdict(result) + subject_id = result.get("subject_id", "-") + agent = result.get("agent_name") or result.get("qa_agent_id") or "-" + stale = " stale" if result.get("stale") is True else "" + line = f"{subject_id}\t{agent}\t{verdict}{stale}" + typer.echo(f"{line}\t{summary}" if summary else line) + + +def _matching_subject_results( + raw_results: object, + *, + agent_id: str, + subject_ids: list[str], + selected_trace_ids: dict[str, str], +) -> list[dict[str, Any]]: + results = _dict_list(raw_results, label="QA results") + expected_subject_ids = {_canonical_id(subject_id) for subject_id in subject_ids} + matched: dict[str, dict[str, Any]] = {} + for result in results: + subject_id = _canonical_id(result.get("subject_id")) + if ( + _canonical_id(result.get("qa_agent_id")) != _canonical_id(agent_id) + or subject_id not in expected_subject_ids + ): + continue + selected_trace_id = selected_trace_ids.get(subject_id) + if ( + selected_trace_id is not None + and _canonical_id(result.get("analysis_trace_id")) != selected_trace_id + ): + continue + matched[subject_id] = result + return [ + matched[canonical_id] + for subject_id in subject_ids + if (canonical_id := _canonical_id(subject_id)) in matched + ] + + +def _all_terminal(results: list[dict[str, Any]], subject_ids: list[str]) -> bool: + statuses = {_canonical_id(result.get("subject_id")): result.get("status") for result in results} + return all( + statuses.get(_canonical_id(subject_id)) in _TERMINAL_STATUSES for subject_id in subject_ids + ) + + +def _result_exit_code(results: list[dict[str, Any]]) -> int: + if any(result.get("status") == "error" for result in results): + return 3 + verdicts = [_result_verdict(result)[0] for result in results] + if any(verdict in {"failed", "unknown"} for verdict in verdicts): + return 1 + if any(verdict != "passed" for verdict in verdicts): + return 3 + return 0 + + +@qa_app.command("agents") +def list_agents( + subject_type: str = typer.Option( + "environment", + "--subject-type", + help="Resource scope: environment or taskset.", + ), + json_output: bool = typer.Option(False, "--json", help="Output the machine-readable response."), + limit: int = typer.Option(50, "--limit", min=1, max=500, help="Maximum agents to return."), + offset: int = typer.Option(0, "--offset", min=0, help="Number of agents to skip."), +) -> None: + """List QA agents available for a resource type.""" + platform = _platform() + normalized_type = _subject_type(subject_type) + try: + response = platform.get( + "/qa-agents", + params={"subject_type": normalized_type, "limit": limit, "offset": offset}, + ) + except (HudNetworkError, HudTimeoutError) as exc: + _execution_error(str(exc)) + except HudRequestError as exc: + _request_error(str(exc)) + if not isinstance(response, dict) or not isinstance(response.get("items"), list): + typer.echo("Platform returned an invalid QA agent list.", err=True) + raise typer.Exit(3) + if json_output: + _print_json(response) + return + agents = _dict_list(response["items"], label="QA agent list") + if not agents: + typer.echo(f"No {normalized_type} QA agents found.") + return + for agent in agents: + typer.echo( + f"{agent.get('id', '-')}\t{agent.get('name', '-')}\t" + f"{agent.get('subject_type', '-')}\t{agent.get('model_name') or '-'}" + ) + + +@qa_app.command("run") +def run_agent( + agent_id: str = typer.Argument(..., help="QA agent UUID."), + subject_ids: list[str] = typer.Argument( # noqa: B008 + ..., + help="One or more Environment or Taskset UUIDs.", + ), + overwrite: bool = typer.Option( + False, + "--overwrite", + help="Create a fresh attempt even when current evidence already exists.", + ), + wait: bool = typer.Option( + True, + "--wait/--no-wait", + help="Wait for every launched analysis to finish.", + ), + timeout: float = typer.Option( + 900, + "--timeout", + min=1, + help="Maximum seconds to wait for QA execution.", + ), + json_output: bool = typer.Option(False, "--json", help="Output machine-readable results."), +) -> None: + """Run one QA agent against Environment or Taskset subjects.""" + platform = _platform() + try: + raw_agent = platform.get(f"/qa-agents/{agent_id}") + if ( + not isinstance(raw_agent, dict) + or raw_agent.get("subject_type") not in _RESOURCE_SUBJECT_TYPES + ): + _execution_error("Platform returned an invalid resource QA agent.") + agent_subject_type = str(raw_agent["subject_type"]) + raw_runs = platform.post( + f"/qa-agents/{agent_id}/run-resources", + json={"subject_ids": subject_ids, "overwrite": overwrite}, + ) + except (HudNetworkError, HudTimeoutError) as exc: + _execution_error(str(exc)) + except HudRequestError as exc: + _request_error(str(exc)) + runs = _dict_list(raw_runs, label="QA launch response") + selected_trace_ids = _selected_trace_ids( + runs, + agent_id=agent_id, + subject_ids=subject_ids, + ) + if not wait: + if json_output: + _print_json(runs) + else: + _render_results(runs) + return + + deadline = time.monotonic() + timeout + results: list[dict[str, Any]] = [] + while time.monotonic() < deadline: + try: + raw_results = platform.get( + "/qa-agents/results/resources", + params={ + "subject_type": agent_subject_type, + "subject_ids": subject_ids, + }, + ) + except (HudNetworkError, HudTimeoutError) as exc: + _execution_error(str(exc)) + except HudRequestError as exc: + _request_error(str(exc)) + results = _matching_subject_results( + raw_results, + agent_id=agent_id, + subject_ids=subject_ids, + selected_trace_ids=selected_trace_ids, + ) + if _all_terminal(results, subject_ids): + break + time.sleep(_POLL_INTERVAL_SECONDS) + else: + if json_output: + _print_json(results) + else: + _render_results(results) + _execution_error(f"Timed out after {timeout:g}s waiting for QA runs.") + + if json_output: + _print_json(results) + else: + _render_results(results) + exit_code = _result_exit_code(results) + if exit_code: + raise typer.Exit(exit_code) + + +@qa_app.command("results") +def list_results( + subject_type: str = typer.Argument(..., help="Resource scope: environment or taskset."), + subject_ids: list[str] = typer.Argument( # noqa: B008 + ..., + help="One or more Environment or Taskset UUIDs.", + ), + json_output: bool = typer.Option(False, "--json", help="Output machine-readable results."), +) -> None: + """Inspect QA results attached to Environment or Taskset subjects.""" + platform = _platform() + normalized_type = _subject_type(subject_type) + try: + raw_results = platform.get( + "/qa-agents/results/resources", + params={"subject_type": normalized_type, "subject_ids": subject_ids}, + ) + except (HudNetworkError, HudTimeoutError) as exc: + _execution_error(str(exc)) + except HudRequestError as exc: + _request_error(str(exc)) + results = _dict_list(raw_results, label="QA results") + if json_output: + _print_json(results) + else: + _render_results(results) diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py new file mode 100644 index 000000000..4b0dabd3a --- /dev/null +++ b/hud/cli/tests/test_qa.py @@ -0,0 +1,415 @@ +"""CLI behavior for resource-scoped platform QA agents.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from hud.cli import app +from hud.utils.exceptions import HudNetworkError, HudRequestError, HudTimeoutError + +runner = CliRunner() + +_AGENT_ID = "00000000-0000-4000-a000-000000000001" +_SUBJECT_ID = "00000000-0000-4000-a000-000000000002" +_SECOND_SUBJECT_ID = "00000000-0000-4000-a000-000000000005" +_TRACE_ID = "00000000-0000-4000-a000-000000000003" +_SECOND_TRACE_ID = "00000000-0000-4000-a000-000000000006" + + +def _agent() -> dict[str, object]: + return { + "id": _AGENT_ID, + "name": "Benchmark Coverage", + "subject_type": "taskset", + "scenario_name": "trace-explorer:taskset_benchmark_coverage", + "model_name": "claude-sonnet", + "public": False, + } + + +def _run( + status: str = "queued", + *, + subject_id: str = _SUBJECT_ID, + analysis_trace_id: str = _TRACE_ID, +) -> dict[str, object]: + return { + "id": "00000000-0000-4000-a000-000000000004", + "qa_agent_id": _AGENT_ID, + "subject_type": "taskset", + "subject_id": subject_id, + "analysis_trace_id": analysis_trace_id, + "status": status, + "attempt": 1, + } + + +def _result(verdict: str = "passed") -> dict[str, object]: + return { + "qa_agent_id": _AGENT_ID, + "subject_type": "taskset", + "subject_id": _SUBJECT_ID, + "agent_name": "Benchmark Coverage", + "analysis_trace_id": _TRACE_ID, + "status": "completed", + "canonical_result": { + "schema_version": "qa_agent_result.v1", + "verdict": verdict, + "summary": "Coverage is sufficient." if verdict == "passed" else "A gap was found.", + "findings": [], + "metadata": {}, + }, + "error": None, + "stale": False, + "attempt": 1, + } + + +def test_qa_agents_lists_resource_agents() -> None: + """Agent discovery forwards subject scope and renders stable identifiers.""" + platform = MagicMock() + platform.get.return_value = { + "items": [_agent()], + "total": 1, + "limit": 50, + "offset": 0, + } + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "agents", "--subject-type", "taskset"]) + + assert result.exit_code == 0 + assert "Benchmark Coverage" in result.output + assert _AGENT_ID in result.output + platform.get.assert_called_once_with( + "/qa-agents", + params={"subject_type": "taskset", "limit": 50, "offset": 0}, + ) + + +def test_qa_agents_json_preserves_platform_payload() -> None: + """Machine output retains pagination and agent fields without reshaping.""" + payload = {"items": [_agent()], "total": 1, "limit": 50, "offset": 0} + platform = MagicMock() + platform.get.return_value = payload + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "agents", "--subject-type", "taskset", "--json"], + ) + + assert result.exit_code == 0 + assert json.loads(result.output) == payload + + +def test_qa_run_reuses_evidence_by_default_and_can_skip_waiting() -> None: + """The default preserves evidence, while --no-wait returns after selection.""" + platform = MagicMock() + platform.get.return_value = _agent() + platform.post.return_value = [_run()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--no-wait"], + ) + + assert result.exit_code == 0 + assert "queued" in result.output + platform.post.assert_called_once_with( + f"/qa-agents/{_AGENT_ID}/run-resources", + json={"subject_ids": [_SUBJECT_ID], "overwrite": False}, + ) + platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") + + +def test_qa_run_waits_for_terminal_result_and_returns_quality_exit() -> None: + """Waiting returns one for a completed quality failure, not an execution error.""" + platform = MagicMock() + platform.post.return_value = [_run()] + platform.get.side_effect = [ + _agent(), + [{**_run(), "status": "queued"}], + [_result("failed")], + ] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + patch("hud.cli.qa.time.sleep"), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--wait"], + ) + + assert result.exit_code == 1 + assert "failed" in result.output.lower() + assert "A gap was found." in result.output + assert platform.get.call_count == 3 + platform.get.assert_called_with( + "/qa-agents/results/resources", + params={"subject_type": "taskset", "subject_ids": [_SUBJECT_ID]}, + ) + + +def test_qa_run_reused_failure_preserves_quality_exit() -> None: + """Run-new-only reuse still evaluates the stored result when waiting.""" + platform = MagicMock() + platform.post.return_value = [_run(status="completed")] + platform.get.side_effect = [_agent(), [_result("failed")]] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "run", _AGENT_ID, _SUBJECT_ID]) + + assert result.exit_code == 1 + assert "failed" in result.output.lower() + assert "A gap was found." in result.output + platform.get.assert_called_with( + "/qa-agents/results/resources", + params={"subject_type": "taskset", "subject_ids": [_SUBJECT_ID]}, + ) + + +def test_qa_run_pins_exact_reused_attempt_from_launch_response() -> None: + """A newer nonmatching attempt cannot replace the exact evidence selected by Platform.""" + selected_failure = _result("failed") + newer_pass = { + **_result("passed"), + "analysis_trace_id": _SECOND_TRACE_ID, + "attempt": 2, + } + platform = MagicMock() + platform.post.return_value = [_run(status="completed")] + platform.get.side_effect = [_agent(), [selected_failure, newer_pass]] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "run", _AGENT_ID, _SUBJECT_ID]) + + assert result.exit_code == 1 + assert "A gap was found." in result.output + + +def test_qa_run_matches_canonical_results_for_uppercase_uuid_input() -> None: + """API-normalized UUID casing does not make a completed result disappear.""" + platform = MagicMock() + platform.post.return_value = [_run(status="completed")] + platform.get.side_effect = [_agent(), [_result("passed")]] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID.upper(), _SUBJECT_ID.upper()], + ) + + assert result.exit_code == 0 + assert "passed" in result.output.lower() + + +def test_qa_run_partial_reuse_waits_for_new_and_scores_all_subjects() -> None: + """A reused failure remains visible while another subject runs.""" + reused_failure = { + **_result("failed"), + "subject_id": _SECOND_SUBJECT_ID, + "analysis_trace_id": _SECOND_TRACE_ID, + } + platform = MagicMock() + platform.post.return_value = [ + _run(), + _run( + status="completed", + subject_id=_SECOND_SUBJECT_ID, + analysis_trace_id=_SECOND_TRACE_ID, + ), + ] + platform.get.side_effect = [ + _agent(), + [{**_run(), "status": "queued"}, reused_failure], + [_result("passed"), reused_failure], + ] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + patch("hud.cli.qa.time.sleep"), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, _SECOND_SUBJECT_ID], + ) + + assert result.exit_code == 1 + assert _SUBJECT_ID in result.output + assert _SECOND_SUBJECT_ID in result.output + platform.get.assert_called_with( + "/qa-agents/results/resources", + params={ + "subject_type": "taskset", + "subject_ids": [_SUBJECT_ID, _SECOND_SUBJECT_ID], + }, + ) + + +def test_qa_run_rejects_incomplete_selected_result_response() -> None: + """Deployment skew fails closed instead of scoring unpinned historical evidence.""" + platform = MagicMock() + platform.get.return_value = _agent() + platform.post.return_value = [_run()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, _SECOND_SUBJECT_ID], + ) + + assert result.exit_code == 3 + assert "exactly one QA result per requested resource" in result.output + assert platform.get.call_count == 1 + + +def test_qa_run_wait_timeout_is_execution_error() -> None: + """An exhausted local wait budget is not reported as a quality failure.""" + platform = MagicMock() + platform.get.return_value = _agent() + platform.post.return_value = [_run()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + patch("hud.cli.qa.time.monotonic", side_effect=[0, 2]), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--wait", "--timeout", "1"], + ) + + assert result.exit_code == 3 + assert "Timed out after 1s" in result.output + platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") + + +def test_qa_run_rejects_missing_analysis_trace_contract() -> None: + """A malformed launch cannot enter a polling loop that never resolves.""" + platform = MagicMock() + run = _run() + del run["analysis_trace_id"] + platform.get.return_value = _agent() + platform.post.return_value = [run] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--wait"]) + + assert result.exit_code == 3 + assert "exactly one QA result per requested resource" in result.output + platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") + + +def test_qa_results_queries_repeated_subject_ids() -> None: + """Result inspection passes the canonical resource scope and identifiers.""" + platform = MagicMock() + platform.get.return_value = [_result()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "results", "taskset", _SUBJECT_ID, "--json"], + ) + + assert result.exit_code == 0 + assert json.loads(result.output)[0]["canonical_result"]["verdict"] == "passed" + platform.get.assert_called_once_with( + "/qa-agents/results/resources", + params={"subject_type": "taskset", "subject_ids": [_SUBJECT_ID]}, + ) + + +def test_qa_results_rejects_trace_scope_before_request() -> None: + """The resource CLI does not route trace subjects through the wrong API.""" + platform = MagicMock() + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "results", "trace", _SUBJECT_ID]) + + assert result.exit_code == 2 + assert "environment, taskset" in result.output + platform.get.assert_not_called() + + +def test_qa_request_failure_uses_request_error_exit() -> None: + """Authentication and platform failures stay distinct from quality verdicts.""" + platform = MagicMock() + platform.get.side_effect = HudRequestError("access denied", status_code=403) + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "agents", "--subject-type", "environment"]) + + assert result.exit_code == 2 + assert "access denied" in result.output + + +def test_qa_network_failure_uses_execution_error_exit() -> None: + """Connection failures are execution errors, not quality failures.""" + platform = MagicMock() + platform.get.side_effect = HudNetworkError("connection failed") + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "agents"]) + + assert result.exit_code == 3 + assert "connection failed" in result.output + + +def test_qa_poll_timeout_failure_uses_execution_error_exit() -> None: + """Transport timeouts during polling preserve the execution-error contract.""" + platform = MagicMock() + platform.get.side_effect = [_agent(), HudTimeoutError("request timed out")] + platform.post.return_value = [_run()] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "run", _AGENT_ID, _SUBJECT_ID]) + + assert result.exit_code == 3 + assert "request timed out" in result.output diff --git a/hud/utils/platform.py b/hud/utils/platform.py index 6184bbf4f..a87f1f155 100644 --- a/hud/utils/platform.py +++ b/hud/utils/platform.py @@ -46,7 +46,7 @@ def base_url(self) -> str: def url(self, path: str, params: dict[str, Any] | None = None) -> str: url = f"{self.base_url}{path}" if params: - url += "?" + urlencode(params) + url += "?" + urlencode(params, doseq=True) return url def get(self, path: str, *, params: dict[str, Any] | None = None) -> Any: diff --git a/hud/utils/tests/test_platform.py b/hud/utils/tests/test_platform.py index b1356614d..0261f0428 100644 --- a/hud/utils/tests/test_platform.py +++ b/hud/utils/tests/test_platform.py @@ -20,6 +20,19 @@ def test_url_prefixes_version_segment_and_joins_params() -> None: ) +def test_url_encodes_repeated_query_parameters() -> None: + """List-valued FastAPI query parameters are emitted as repeated keys.""" + platform = PlatformClient("https://api.example", "key") + + assert platform.url( + "/qa-agents/results/resources", + {"subject_type": "taskset", "subject_ids": ["first", "second"]}, + ) == ( + "https://api.example/v2/qa-agents/results/resources?" + "subject_type=taskset&subject_ids=first&subject_ids=second" + ) + + def test_get_and_post_route_through_shared_requests(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[dict[str, object]] = [] From 81c7394730700212e224b9ca12259c5261ca92ba Mon Sep 17 00:00:00 2001 From: CK0607 Date: Mon, 3 Aug 2026 13:41:06 +0530 Subject: [PATCH 2/3] fix(cli): render verdicts for reused QA results Use the result payload returned by resource selection so no-wait output does not hide a stored quality verdict behind the terminal execution status. Co-authored-by: Cursor --- hud/cli/qa.py | 15 ++++++++------- hud/cli/tests/test_qa.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/hud/cli/qa.py b/hud/cli/qa.py index 96d02b8d6..2bca90984 100644 --- a/hud/cli/qa.py +++ b/hud/cli/qa.py @@ -90,13 +90,14 @@ def _selected_trace_ids( def _result_verdict(result: dict[str, Any]) -> tuple[str, str | None]: - canonical = result.get("canonical_result") - if isinstance(canonical, dict): - canonical_dict = cast("dict[str, Any]", canonical) - verdict = canonical_dict.get("verdict") - summary = canonical_dict.get("summary") - if isinstance(verdict, str): - return verdict, summary if isinstance(summary, str) else None + for result_field in ("canonical_result", "result"): + canonical = result.get(result_field) + if isinstance(canonical, dict): + canonical_dict = cast("dict[str, Any]", canonical) + verdict = canonical_dict.get("verdict") + summary = canonical_dict.get("summary") + if isinstance(verdict, str): + return verdict, summary if isinstance(summary, str) else None status = result.get("status") error = result.get("error") return ( diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py index 4b0dabd3a..044b9a439 100644 --- a/hud/cli/tests/test_qa.py +++ b/hud/cli/tests/test_qa.py @@ -136,6 +136,36 @@ def test_qa_run_reuses_evidence_by_default_and_can_skip_waiting() -> None: platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") +def test_qa_run_no_wait_renders_reused_result_verdict() -> None: + """A terminal selection reports its stored verdict without an extra result request.""" + platform = MagicMock() + platform.get.return_value = _agent() + platform.post.return_value = [ + { + **_run(status="completed"), + "result": { + "schema_version": "qa_agent_result.v1", + "verdict": "failed", + "summary": "A gap was found.", + }, + }, + ] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--no-wait"], + ) + + assert result.exit_code == 0 + assert "failed" in result.output + assert "A gap was found." in result.output + platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") + + def test_qa_run_waits_for_terminal_result_and_returns_quality_exit() -> None: """Waiting returns one for a completed quality failure, not an execution error.""" platform = MagicMock() From f896d0f20fb92c3bd69fde2aad6e84f8ffb6e262 Mon Sep 17 00:00:00 2001 From: CK0607 Date: Mon, 3 Aug 2026 15:19:23 +0530 Subject: [PATCH 3/3] fix(cli): reject unsupported QA agent scopes Classify valid non-resource agents as unsupported caller input instead of a platform execution failure. Co-authored-by: Cursor --- hud/cli/qa.py | 9 ++++----- hud/cli/tests/test_qa.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/hud/cli/qa.py b/hud/cli/qa.py index 2bca90984..bf4589278 100644 --- a/hud/cli/qa.py +++ b/hud/cli/qa.py @@ -237,12 +237,11 @@ def run_agent( platform = _platform() try: raw_agent = platform.get(f"/qa-agents/{agent_id}") - if ( - not isinstance(raw_agent, dict) - or raw_agent.get("subject_type") not in _RESOURCE_SUBJECT_TYPES - ): + if not isinstance(raw_agent, dict) or not isinstance(raw_agent.get("subject_type"), str): _execution_error("Platform returned an invalid resource QA agent.") - agent_subject_type = str(raw_agent["subject_type"]) + agent_subject_type = raw_agent["subject_type"] + if agent_subject_type not in _RESOURCE_SUBJECT_TYPES: + _request_error("QA agent must target environment or taskset subjects.") raw_runs = platform.post( f"/qa-agents/{agent_id}/run-resources", json={"subject_ids": subject_ids, "overwrite": overwrite}, diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py index 044b9a439..cbcf172ce 100644 --- a/hud/cli/tests/test_qa.py +++ b/hud/cli/tests/test_qa.py @@ -136,6 +136,25 @@ def test_qa_run_reuses_evidence_by_default_and_can_skip_waiting() -> None: platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") +def test_qa_run_rejects_a_non_resource_agent_as_caller_input() -> None: + """A valid trace agent is unsupported input, not a broken Platform response.""" + platform = MagicMock() + platform.get.return_value = {**_agent(), "subject_type": "trace"} + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--no-wait"], + ) + + assert result.exit_code == 2 + assert "must target environment or taskset" in result.output + platform.post.assert_not_called() + + def test_qa_run_no_wait_renders_reused_result_verdict() -> None: """A terminal selection reports its stored verdict without an extra result request.""" platform = MagicMock()