diff --git a/hud/cli/qa.py b/hud/cli/qa.py index bf4589278..8625a02ce 100644 --- a/hud/cli/qa.py +++ b/hud/cli/qa.py @@ -13,7 +13,7 @@ from hud.utils.platform import PlatformClient _POLL_INTERVAL_SECONDS = 2.0 -_RESOURCE_SUBJECT_TYPES = {"environment", "taskset"} +_RESOURCE_SUBJECT_TYPES = {"environment", "taskset", "task"} _TERMINAL_STATUSES = {"completed", "error"} qa_app = typer.Typer( @@ -173,7 +173,7 @@ def list_agents( subject_type: str = typer.Option( "environment", "--subject-type", - help="Resource scope: environment or taskset.", + help="Resource scope: environment, taskset, or task.", ), json_output: bool = typer.Option(False, "--json", help="Output the machine-readable response."), limit: int = typer.Option(50, "--limit", min=1, max=500, help="Maximum agents to return."), @@ -213,7 +213,7 @@ def run_agent( agent_id: str = typer.Argument(..., help="QA agent UUID."), subject_ids: list[str] = typer.Argument( # noqa: B008 ..., - help="One or more Environment or Taskset UUIDs.", + help="One or more Environment, Taskset, or Task UUIDs.", ), overwrite: bool = typer.Option( False, @@ -233,7 +233,7 @@ def run_agent( ), json_output: bool = typer.Option(False, "--json", help="Output machine-readable results."), ) -> None: - """Run one QA agent against Environment or Taskset subjects.""" + """Run one QA agent against Environment, Taskset, or Task subjects.""" platform = _platform() try: raw_agent = platform.get(f"/qa-agents/{agent_id}") @@ -241,7 +241,7 @@ def run_agent( _execution_error("Platform returned an invalid resource QA agent.") agent_subject_type = raw_agent["subject_type"] if agent_subject_type not in _RESOURCE_SUBJECT_TYPES: - _request_error("QA agent must target environment or taskset subjects.") + _request_error("QA agent must target environment, taskset, or task subjects.") raw_runs = platform.post( f"/qa-agents/{agent_id}/run-resources", json={"subject_ids": subject_ids, "overwrite": overwrite}, @@ -261,6 +261,11 @@ def run_agent( _print_json(runs) else: _render_results(runs) + terminal_results = [result for result in runs if result.get("status") in _TERMINAL_STATUSES] + if terminal_results: + exit_code = _result_exit_code(terminal_results) + if exit_code: + raise typer.Exit(exit_code) return deadline = time.monotonic() + timeout @@ -305,14 +310,17 @@ def run_agent( @qa_app.command("results") def list_results( - subject_type: str = typer.Argument(..., help="Resource scope: environment or taskset."), + subject_type: str = typer.Argument( + ..., + help="Resource scope: environment, taskset, or task.", + ), subject_ids: list[str] = typer.Argument( # noqa: B008 ..., - help="One or more Environment or Taskset UUIDs.", + help="One or more Environment, Taskset, or Task UUIDs.", ), json_output: bool = typer.Option(False, "--json", help="Output machine-readable results."), ) -> None: - """Inspect QA results attached to Environment or Taskset subjects.""" + """Inspect QA results attached to Environment, Taskset, or Task subjects.""" platform = _platform() normalized_type = _subject_type(subject_type) try: diff --git a/hud/cli/tests/test_qa.py b/hud/cli/tests/test_qa.py index cbcf172ce..9b90a36a0 100644 --- a/hud/cli/tests/test_qa.py +++ b/hud/cli/tests/test_qa.py @@ -112,6 +112,29 @@ def test_qa_agents_json_preserves_platform_payload() -> None: assert json.loads(result.output) == payload +def test_qa_agents_accepts_task_scope() -> None: + """Task is a first-class resource scope for discovery.""" + platform = MagicMock() + platform.get.return_value = { + "items": [{**_agent(), "subject_type": "task"}], + "total": 1, + "limit": 50, + "offset": 0, + } + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke(app, ["qa", "agents", "--subject-type", "task"]) + + assert result.exit_code == 0 + platform.get.assert_called_once_with( + "/qa-agents", + params={"subject_type": "task", "limit": 50, "offset": 0}, + ) + + def test_qa_run_reuses_evidence_by_default_and_can_skip_waiting() -> None: """The default preserves evidence, while --no-wait returns after selection.""" platform = MagicMock() @@ -151,10 +174,32 @@ def test_qa_run_rejects_a_non_resource_agent_as_caller_input() -> None: ) assert result.exit_code == 2 - assert "must target environment or taskset" in result.output + assert "must target environment, taskset, or task" in result.output platform.post.assert_not_called() +def test_qa_run_accepts_task_agent() -> None: + """Task agents use the native resource run endpoint without trace indirection.""" + platform = MagicMock() + platform.get.return_value = {**_agent(), "subject_type": "task"} + platform.post.return_value = [{**_run(), "subject_type": "task"}] + + with ( + patch("hud.cli.qa.require_api_key", return_value="api-key"), + patch("hud.cli.qa.PlatformClient.from_settings", return_value=platform), + ): + result = runner.invoke( + app, + ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--no-wait"], + ) + + assert result.exit_code == 0 + platform.post.assert_called_once_with( + f"/qa-agents/{_AGENT_ID}/run-resources", + json={"subject_ids": [_SUBJECT_ID], "overwrite": False}, + ) + + def test_qa_run_no_wait_renders_reused_result_verdict() -> None: """A terminal selection reports its stored verdict without an extra result request.""" platform = MagicMock() @@ -179,7 +224,7 @@ def test_qa_run_no_wait_renders_reused_result_verdict() -> None: ["qa", "run", _AGENT_ID, _SUBJECT_ID, "--no-wait"], ) - assert result.exit_code == 0 + assert result.exit_code == 1 assert "failed" in result.output assert "A gap was found." in result.output platform.get.assert_called_once_with(f"/qa-agents/{_AGENT_ID}") @@ -414,7 +459,7 @@ def test_qa_results_rejects_trace_scope_before_request() -> None: result = runner.invoke(app, ["qa", "results", "trace", _SUBJECT_ID]) assert result.exit_code == 2 - assert "environment, taskset" in result.output + assert "environment, task, taskset" in result.output platform.get.assert_not_called()