From 05216bda3723bc9b5e00b9e4ce7ec3580e3d1e78 Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Tue, 17 Nov 2026 15:01:22 +0700 Subject: [PATCH] feat(gooddata-eval): label harness errors, keep every verdict QA-30213, phase 1: a failing run is labelled ERROR only when the test harness failed (its simulated user or its judge), FAIL otherwise. The rule is fail-closed, in core/outcome.py; chat_error, agent_timeout, turn_incomplete and any unknown cause stay FAIL, since a customer hits them too. A judge failure is an ERROR only when the judge's verdict was the one still open: a document run that already failed a fixed check stays a FAIL whatever the judge would have said. The labels decide nothing: an ERROR run still counts as a run that did not pass, so gates, pass rates and scores are unchanged. They appear in the "Failing runs" part of the failure message, in failed_runs[] (exit_reason, outcome, failed_checks), on the raised exception or the returned result (harness_error when every run errored, and shadow, the verdict a phase-2 candidate rule would give, recorded only for QA-30296), in the gd-eval run JSON report, and in Langfuse as an extra eval_error score. Shadow is recorded on passing items too: under pass@K an item can pass with most of its runs errored. An errored run that gets no verdict is attached to its experiment item whether its trace joined or not, as every scored run is. An ERROR run still lists its failed checks, because the simulated user is also asked after an answer that was already final. An item whose every run errored says so first, and says no run was scored only when every judge failed. The gate note names errored runs; under pass^K an item is still called unstable when a run the agent failed sits beside a passing one. A run with no chart, metric or alert names only that, not every check that reads False without it. KDA, anomaly, what-if, general question and guardrail name their failed checks as well. The failing runs are cut at 2000 characters, at a run boundary; every run stays in failed_runs[], whose outcome is read from the same labels as the message and the eval_error score. Judge failures of general question, guardrail and document skill are labelled too; a failed takeaway request stays FAIL. A timed-out or incomplete turn records agent_timeout / turn_incomplete instead of chat_error, including an agent that goes silent past its budget and ends in a read timeout, whether mid-stream or awaiting the headers. jira: QA-30213 risk: low Co-Authored-By: Claude Opus 5.5 --- packages/gooddata-eval/AGENTS.md | 1 + .../src/gooddata_eval/cli/agentic_runner.py | 13 + .../core/agentic/_failed_runs.py | 72 ++- .../core/agentic/_failure_message.py | 198 +++++++ .../src/gooddata_eval/core/agentic/_gate.py | 13 +- .../gooddata_eval/core/agentic/_outcome.py | 201 ++++++- .../core/agentic/_trace_linker.py | 5 +- .../gooddata_eval/core/agentic/alert_skill.py | 35 +- .../core/agentic/anomaly_detection.py | 55 +- .../core/agentic/conversation.py | 12 +- .../core/agentic/dashboard_skill.py | 37 +- .../core/agentic/document_skill.py | 28 +- .../core/agentic/general_question.py | 37 +- .../gooddata_eval/core/agentic/guardrail.py | 37 +- .../gooddata_eval/core/agentic/kda_skill.py | 44 +- .../core/agentic/metric_skill.py | 46 +- .../core/agentic/visualization.py | 113 +++- .../src/gooddata_eval/core/agentic/what_if.py | 59 +- .../src/gooddata_eval/core/chat/sse_client.py | 51 +- .../core/evaluators/_llm_judge.py | 4 + .../src/gooddata_eval/core/models.py | 36 +- .../src/gooddata_eval/core/outcome.py | 159 ++++++ .../core/reporting/json_report.py | 3 + .../src/gooddata_eval/core/runner.py | 5 + .../tests/test_agentic_alert_skill.py | 20 +- .../tests/test_agentic_anomaly_detection.py | 15 +- .../tests/test_agentic_conversation.py | 28 +- .../tests/test_agentic_dashboard_skill.py | 18 +- .../tests/test_agentic_document_skill.py | 44 +- .../tests/test_agentic_failure_message.py | 522 +++++++++++++++++ .../gooddata-eval/tests/test_agentic_gate.py | 16 + .../tests/test_agentic_kda_skill.py | 30 +- .../tests/test_agentic_metric_skill.py | 80 ++- .../tests/test_agentic_observe_experiment.py | 12 + .../tests/test_agentic_runner.py | 34 +- .../tests/test_agentic_visualization.py | 34 +- .../tests/test_agentic_what_if.py | 15 +- .../tests/test_eval_error_scores.py | 162 ++++++ packages/gooddata-eval/tests/test_outcome.py | 539 ++++++++++++++++++ .../gooddata-eval/tests/test_sse_client.py | 67 +++ 40 files changed, 2751 insertions(+), 149 deletions(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/outcome.py create mode 100644 packages/gooddata-eval/tests/test_agentic_failure_message.py create mode 100644 packages/gooddata-eval/tests/test_eval_error_scores.py create mode 100644 packages/gooddata-eval/tests/test_outcome.py diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 183cb387c..4b33b57a2 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -34,6 +34,7 @@ experiment. The newest and most actively developed package in the repo. | `core/reporting/` | console and JSON output rendering | | `core/scoring.py`, `core/runner.py` | scoring and orchestration | | `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends | +| `core/outcome.py` | `Outcome` (PASS / FAIL / ERROR): the one fail-closed rule that labels a failing run ERROR when the harness failed (simulated user, judge), plus the item label and the recorded-only shadow verdict. Labels decide no verdict; the agentic kinds reach it through `core/agentic/_outcome.py`'s `label_runs` | **Depends on**: `gooddata-sdk`, `httpx`, `pydantic`, `orjson`, `rich`. The LLM-judge evaluator is an optional extra (`llm-judge`, pulling `openai>=1.45,<2.0`); every `openai` diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index da4619008..e596c6128 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -404,6 +404,16 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None: item_report.runs_ungraded = unscored +def _apply_labels(item_report: ItemReport, source: Any) -> None: + """Copy the item's harness label and shadow verdict off an outcome or a failure, if set.""" + harness_error = getattr(source, "harness_error", None) + if isinstance(harness_error, dict): + item_report.harness_error = harness_error + shadow = getattr(source, "shadow", None) + if isinstance(shadow, dict): + item_report.shadow = shadow + + def _apply_failed_runs(item_report: ItemReport, source: Any) -> None: """Copy the per-run failure records off an outcome or a failure, if the kind built any. @@ -555,6 +565,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: _apply_best_run_latency(item_report, outcome) _apply_run_counts(item_report, outcome) _apply_failed_runs(item_report, outcome) + _apply_labels(item_report, outcome) _apply_run_latencies(item_report, outcome) except AssertionError as exc: item_report.gate_passed = False if gated else None @@ -567,6 +578,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: _apply_best_run_latency(item_report, exc) _apply_run_counts(item_report, exc) _apply_failed_runs(item_report, exc) + _apply_labels(item_report, exc) _apply_run_latencies(item_report, exc) # Read off the counts, not off the gate: pass^K fails items where runs did pass, # and reporting those as pass_at_k False would contradict the Langfuse score of @@ -588,6 +600,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: _apply_timings(item_report, getattr(exc, "timings", None)) _apply_run_counts(item_report, exc) _apply_failed_runs(item_report, exc) + _apply_labels(item_report, exc) _apply_run_latencies(item_report, exc) # Every run ungraded, by definition of this error -- said explicitly rather than # left to _apply_run_counts, whose source is detail["unscored_runs"] and which a diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py index 694928b9b..08bd4f899 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py @@ -25,17 +25,28 @@ from collections.abc import Callable, Sequence from typing import Any +from gooddata_eval.core.agentic._failure_message import cap_failed_checks +from gooddata_eval.core.agentic._outcome import RunLabels, run_exit_reason, run_outcome from gooddata_eval.core.models import build_tool_calls +from gooddata_eval.core.outcome import OutcomeStatus def build_failed_runs( run_results: Sequence[Any], *, - passed: Callable[[Any], bool], detail: Callable[[Any], dict], + labels: RunLabels | None = None, + passed: Callable[[Any], bool] | None = None, + failed_checks: Callable[[Any], list[dict]] | None = None, ) -> list[dict]: """One record per run that did not pass, in the order the runs happened. + A kind that labels its runs passes its ``labels`` (from ``label_runs``, in run order): + they say which runs did not pass and are the ``outcome`` of each record, so the records, + the failure message and the Langfuse ``eval_error`` marker read one set of labels. A kind + that does not label its runs passes ``passed`` instead, and each record's ``outcome`` is + worked out here from the run alone. Exactly one of the two is given. + ``passed`` and ``detail`` are supplied by the caller because neither is uniform across kinds: guardrail reads ``run.passed`` while visualization reads ``run.eval_result.strict_pass``, and each kind's diagnostic dict is its own. Everything @@ -45,28 +56,51 @@ def build_failed_runs( ``ToolCallEvent``s, so a run that has the list must hold real events in it. A run the judge could not grade is recorded like any other non-passing run, with its - ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the - verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``. + ``error`` set to the run's ``judge_error``. That matches ``core/runner.py``, which also + records ungraded runs; the verdict-level accounting of ungraded runs stays with the + caller's ``unscored_runs``. + + ``exit_reason`` is the run's ``LoopExit`` value, or null for a kind whose loop records + none, and ``outcome`` its ``core.outcome.Outcome`` record: ERROR when the harness failed + (its simulated user, or its judge when the judge's verdict was the one still open), FAIL + otherwise. ``error`` and ``outcome`` can differ: a run that failed a fixed check and whose + judge also broke has ``error`` set and is a FAIL. The label decides nothing; the run counts + as not passed either way. + + ``failed_checks`` is written only when the caller supplies it: a missing key means the + kind does not name its failing checks, while an empty list would claim the run failed + none. Its expected/actual values are capped in size; the run's ``detail`` keeps them whole. """ + if (labels is None) == (passed is None): + raise ValueError("build_failed_runs takes exactly one of labels and passed") + if labels is not None and len(labels.outcomes) != len(run_results): + raise ValueError(f"{len(labels.outcomes)} labels for {len(run_results)} runs") records: list[dict] = [] for run_index, run in enumerate(run_results, start=1): - if passed(run): + if labels is not None: + outcome = labels.outcomes[run_index - 1] + else: + outcome = run_outcome(run, passed=passed is not None and passed(run)) + if outcome.status is OutcomeStatus.PASS: continue tool_calls = list(getattr(run, "tool_call_events", None) or []) reasoning_steps = list(getattr(run, "reasoning_steps", None) or []) - records.append( - { - "run_index": run_index, - "passed": False, - "error": getattr(run, "judge_error", None), - "detail": detail(run), - "conversation_id": getattr(run, "conversation_id", None), - "response_id": getattr(run, "response_id", None), - "reasoning_step_count": len(reasoning_steps), - "reasoning_steps": reasoning_steps, - "tool_call_count": len(tool_calls), - "tool_names": [getattr(e, "function_name", None) for e in tool_calls], - "tool_calls": build_tool_calls(tool_calls), - } - ) + record: dict[str, Any] = { + "run_index": run_index, + "passed": False, + "error": getattr(run, "judge_error", None), + "detail": detail(run), + "conversation_id": getattr(run, "conversation_id", None), + "response_id": getattr(run, "response_id", None), + "reasoning_step_count": len(reasoning_steps), + "reasoning_steps": reasoning_steps, + "tool_call_count": len(tool_calls), + "tool_names": [getattr(e, "function_name", None) for e in tool_calls], + "tool_calls": build_tool_calls(tool_calls), + "exit_reason": run_exit_reason(run), + "outcome": outcome.as_record(), + } + if failed_checks is not None: + record["failed_checks"] = cap_failed_checks(failed_checks(run)) + records.append(record) return records diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py new file mode 100644 index 000000000..d05a93087 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py @@ -0,0 +1,198 @@ +# (C) 2026 GoodData Corporation +"""The head of an agentic kind's failure message: the gate result, then every failing run. + +Each kind's assertion message describes its BEST run, which on a partial gate failure (some +of the K runs passed) is a passing run: every check reads True and the reason the item +failed is nowhere in the message. This module renders the part that is missing -- for each +failing run its index, why its loop stopped and the checks it failed, with expected/actual +where the kind knows them -- from the same ``failed_runs`` records the item reports, so the +message and the triage data cannot disagree about which runs failed or why. + +One renderer for every kind that labels its runs, on purpose: the readers of these messages +(people and the triage tooling) look for one shape, whatever the capability under test. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from gooddata_eval.core.outcome import UNKNOWN_REASON, OutcomeStatus + +__all__ = [ + "FailedCheck", + "cap_failed_checks", + "failed_check_records", + "failed_checks_from", + "failing_runs_note", + "failure_head", + "judge_failed_checks", +] + +# A value longer than this is cut: the message is one line per run, and the whole value is in +# the run's detail anyway. +_MAX_VALUE_CHARS = 160 + +# The failing runs together are cut past this, at a run boundary: K runs of a kind with a dozen +# checks would otherwise put tens of thousands of characters into one assertion message. Every +# run is in ``failed_runs`` whatever the message shows. +_MAX_FAILING_RUNS_CHARS = 2000 + + +class _Unset(Enum): + UNSET = "unset" + + +_UNSET = _Unset.UNSET + + +@dataclass(frozen=True) +class FailedCheck: + """One check a run failed, with the expected/actual pair when the kind knows it. + + ``part`` narrows the check when one name covers several parts, e.g. the ``date`` part of + ``filters_correct``. + """ + + name: str + expected: Any = _UNSET + actual: Any = _UNSET + part: str | None = None + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into a failing run's ``failed_checks``.""" + record: dict[str, Any] = {"name": self.name} + if self.part is not None: + record["part"] = self.part + if self.expected is not _UNSET: + record["expected"] = self.expected + if self.actual is not _UNSET: + record["actual"] = self.actual + return record + + +def failed_check_records(checks: Iterable[FailedCheck]) -> list[dict[str, Any]]: + """``checks`` as the records ``failed_runs[].failed_checks`` holds.""" + return [check.as_record() for check in checks] + + +def failed_checks_from(checks: Mapping[str, bool], *, prerequisite: str | None = None) -> list[dict[str, Any]]: + """The records of the checks in ``checks`` that read False, by name, in their order. + + ``prerequisite`` names the check every other one depends on, e.g. ``alert_created``. When it + failed, the others read False only because there was nothing to check, so it alone is named. + """ + if prerequisite is not None and checks.get(prerequisite) is False: + return failed_check_records([FailedCheck(prerequisite)]) + return failed_check_records(FailedCheck(name) for name, ok in checks.items() if not ok) + + +def judge_failed_checks(name: str, *, passed: bool, judge_error: str | None, reasoning: Any) -> list[dict[str, Any]]: + """The judge's verdict ``name`` on a run it failed, with its reasoning as the actual value. + + Empty when the judge passed the run, or could not grade it: a run with no verdict failed no + judge check. + """ + if judge_error is not None or passed: + return [] + return failed_check_records([FailedCheck(name, part="judge", actual=reasoning)]) + + +def _text(value: Any) -> str: + text = value if isinstance(value, str) else json.dumps(value, default=str, ensure_ascii=False, sort_keys=True) + return " ".join(text.split()) + + +def _value(value: Any) -> str: + if value is None or value == [] or value == {} or value == "": + return "none" + text = _text(value) + return text if len(text) <= _MAX_VALUE_CHARS else text[: _MAX_VALUE_CHARS - 3] + "..." + + +def cap_failed_checks(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """``records`` with every expected/actual value cut to what the message shows of it. + + A check's values can be whole filter or bucket structures, repeated for every failing run + of every item; the record keeps the same ``_MAX_VALUE_CHARS`` the message does. A value + that fits stays as it is, JSON and all; a longer one becomes its cut text. The run's + ``detail`` holds the full values. + """ + capped: list[dict[str, Any]] = [] + for record in records: + entry = dict(record) + for key in ("expected", "actual"): + if key in entry and len(_text(entry[key])) > _MAX_VALUE_CHARS: + entry[key] = _value(entry[key]) + capped.append(entry) + return capped + + +def _render_check(record: Mapping[str, Any]) -> str: + name = str(record.get("name", "?")) + parts: list[str] = [] + if "expected" in record: + parts.append(f"expected {_value(record['expected'])}") + if "actual" in record: + parts.append(f"actual {_value(record['actual'])}") + part = record.get("part") + if not parts and part is None: + return name + inner = ", ".join(parts) + if part is not None: + inner = f"{part}: {inner}" if inner else str(part) + return f"{name}[{inner}]" + + +def _render_run(record: Mapping[str, Any], runs_total: int) -> str: + outcome = record.get("outcome") + head = f"run {record.get('run_index', '?')}/{runs_total}" + exit_reason = record.get("exit_reason") + errored = isinstance(outcome, Mapping) and outcome.get("status") == OutcomeStatus.ERROR.value + if errored: + # A harness error: the reason (simulated_user_failed, judge_failed) is the diagnosis. + # Its checks are still listed: the simulated user is also asked after an answer that + # was already final, and what that answer missed is the only way to tell. + head += f" ERROR ({outcome.get('reason') or UNKNOWN_REASON})" + else: + head += " FAIL" + (f" (loop exit: {exit_reason})" if exit_reason else "") + checks = record.get("failed_checks") + if checks: + return f"{head}: {', '.join(_render_check(c) for c in checks)}" + if checks is not None and not errored: + return f"{head}: no failing check recorded" + return head + + +def failing_runs_note(failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """Every failing run, joined with ``; ``. Empty when no run failed. + + Past ``_MAX_FAILING_RUNS_CHARS`` the runs left are counted instead of rendered. The first + run is always shown, cut to that length if it alone is longer. + """ + shown: list[str] = [] + length = 0 + for index, record in enumerate(failed_runs): + text = _render_run(record, runs_total) + if shown and length + len("; ") + len(text) > _MAX_FAILING_RUNS_CHARS: + more = len(failed_runs) - index + shown.append(f"{more} more failing run(s) in failed_runs") + break + if not shown and len(text) > _MAX_FAILING_RUNS_CHARS: + text = text[: _MAX_FAILING_RUNS_CHARS - 3] + "..." + shown.append(text) + length += len(text) + (len("; ") if len(shown) > 1 else 0) + return "; ".join(shown) + + +def failure_head(gate_note: str, failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """``gate_note`` followed by the failing runs, ready to start a kind's assertion message. + + Returns ``gate_note`` unchanged when there is no failing run to list, which is the whole + head of a kind that builds no ``failed_runs``. + """ + note = failing_runs_note(failed_runs, runs_total) + return f"{gate_note} Failing runs: {note}." if note else gate_note diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py index 63980bb07..ec9ace87f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py @@ -29,20 +29,27 @@ def gate_label(gate: str | None, k: int) -> str: return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}" -def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0) -> str: +def gate_failure_note( + gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0, *, runs_errored: int = 0 +) -> str: """Gate and how many runs met it, for the assertion message. Needed because the message body describes the BEST run, which under pass^K can be a run that passed — so the reported detail on its own looks like a pass. An ungraded run counts in runs_total but can never count in runs_passed, so the remainder - is not evidence of instability — saying so would blame the agent for a judge outage. + is not evidence of instability — saying so would blame the agent for a judge outage. A run + the harness failed (``runs_errored``, labelled ERROR in ``core/outcome.py``) is no evidence + of instability either, so it is named rather than called one. A pass^K item with a passing + run and a run the agent itself failed is unstable all the same, whatever else errored. """ label = gate_label(gate, runs_total) note = f"Gate {label} failed: {runs_passed}/{runs_total} runs passed" if runs_ungraded: return f"{note}, {runs_ungraded} ungraded." - if normalize_gate(gate) == "power" and runs_passed: + if runs_errored: + note = f"{note}, {runs_errored} harness error(s)" + if normalize_gate(gate) == "power" and runs_passed and runs_passed + runs_errored < runs_total: return f"{note} — unstable, not a clean failure." return f"{note}." diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py index 576aee838..11df14b36 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py @@ -1,5 +1,5 @@ # (C) 2026 GoodData Corporation -"""The common tail of every ``evaluate_agentic_*`` function. +"""The common tail of every ``evaluate_agentic_*`` function, and the run labels it carries. Before this module, each of the agentic evaluators hand-wrote the same block: copy ``reasoning_steps``/``conversation_id``/``response_id``/``detail``/``runs_passed``/ @@ -15,22 +15,200 @@ ``timeline_detail`` builds both ``latency_breakdown`` and ``tool_calls`` from the same events so they stay index-aligned, but roughly half the evaluators called ``build_latency_breakdown`` directly and silently never got a ``tool_calls`` key. + +The run labels (``core/outcome.py``) are applied here for the same reason: ``label_runs`` +labels an item's runs once, and the failure message, the Langfuse marker and the fields on +the raised or returned result all read that one ``RunLabels``. ``chat_failure_exit`` names +the loop exit of a run whose chat call raised, for every kind alike. """ from __future__ import annotations +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum from typing import Any, NoReturn +from gooddata_eval.core.agentic._failure_message import failure_head +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, + LoopExit, ReasoningStepEvent, ToolCallEvent, timeline_detail, ) +from gooddata_eval.core.outcome import ( + EVAL_ERROR_SCORE, + JUDGE_FAILED, + Outcome, + classify_run, + item_outcome, + shadow_verdict, +) from gooddata_eval.core.timing import PhaseTimings -__all__ = ["agentic_detail", "agentic_success", "raise_agentic_failure"] +__all__ = [ + "CHAT_FAILURE_EXITS", + "EVAL_ERROR_SCORE", + "RunLabels", + "agentic_detail", + "agentic_success", + "attach_labels", + "chat_failure_exit", + "failure_message_head", + "label_runs", + "log_eval_error_score", + "mark_unscored_run", + "raise_agentic_failure", + "run_exit_reason", + "run_outcome", +] + + +CHAT_FAILURE_EXITS = frozenset({LoopExit.CHAT_ERROR, LoopExit.AGENT_TIMEOUT, LoopExit.TURN_INCOMPLETE}) +"""Every exit ``chat_failure_exit`` can return: the loop stopped because a chat call raised.""" + + +def chat_failure_exit(exc: BaseException) -> LoopExit: + """The ``LoopExit`` for a run whose chat call raised ``exc``. + + ``TurnTimeoutError`` and ``TurnIncompleteError`` subclass ``ChatError`` but are the + agent's own failures -- too slow, or no final answer -- so they get exits of their own + instead of reading as a chat API fault. Anything else a kind catches there (a gen-ai + error event, a broken stream, a 5xx after its retries) is ``CHAT_ERROR``. All three are + FAIL in ``core/outcome.py``: a customer hits each of them too. + """ + if isinstance(exc, TurnTimeoutError): + return LoopExit.AGENT_TIMEOUT + if isinstance(exc, TurnIncompleteError): + return LoopExit.TURN_INCOMPLETE + return LoopExit.CHAT_ERROR + + +def run_exit_reason(run: Any) -> str | None: + """The run's ``exit_reason`` as a string, or None for a kind whose run records none.""" + value = getattr(run, "exit_reason", None) + if isinstance(value, Enum): + return str(value.value) + return value if isinstance(value, str) else None + + +JudgeErrorOf = Callable[[Any], str | None] +"""How a kind reads the judge failure that left a run without a verdict, or None. + +The default reads ``run.judge_error``, right for a kind whose judge is its only check. A kind +with other checks returns the judge's error only when the judge's verdict was the one still +open, and leaves out a failure its ``judge_error`` reports from the product path.""" + + +def _default_judge_error(run: Any) -> str | None: + value = getattr(run, "judge_error", None) + return value if isinstance(value, str) else None + + +def run_outcome(run: Any, *, passed: bool, judge_error: JudgeErrorOf | None = None) -> Outcome: + """One run's label: ``classify_run`` over the run's exit and its judge failure, if any.""" + error = (judge_error or _default_judge_error)(run) + return classify_run(run_exit_reason(run), passed=passed, judge_error=error if isinstance(error, str) else None) + + +@dataclass(frozen=True) +class RunLabels: + """The label of every run of an item, in run order. + + The labels decide nothing: the kind gates on its own pass_at_k/pass_power_k, and an + ERROR run counts as a run that did not pass. They only say why a run did not pass, and + whether the whole item is a harness error (``item_outcome``). + """ + + outcomes: tuple[Outcome, ...] + + def error_label(self, run_index: int) -> Outcome | None: + """The ERROR label of run ``run_index`` (0-based), or None when that run is not one.""" + outcome = self.outcomes[run_index] if 0 <= run_index < len(self.outcomes) else None + return outcome if outcome is not None and outcome.is_error else None + + @property + def item_outcome(self) -> Outcome | None: + """ERROR when every run errored, else None (see ``core.outcome.item_outcome``).""" + return item_outcome(self.outcomes) + + @property + def harness_error_record(self) -> dict[str, Any] | None: + """``item_outcome`` as the ``harness_error`` record a failed item carries, or None.""" + outcome = self.item_outcome + return outcome.as_record() if outcome is not None else None + + def shadow(self, gate: str | None) -> dict[str, Any]: + """The phase-2 candidate verdict, recorded on every item, passed or failed, and never used.""" + return shadow_verdict(self.outcomes, gate) + + +def label_runs( + run_results: Sequence[Any], *, passed: Callable[[Any], bool], judge_error: JudgeErrorOf | None = None +) -> RunLabels: + """The ``RunLabels`` of an item's runs; ``passed`` is the kind's own per-run verdict.""" + return RunLabels(tuple(run_outcome(r, passed=passed(r), judge_error=judge_error) for r in run_results)) + + +def attach_labels(exc: AgenticAssertionError | JudgeResponseError, labels: RunLabels, gate: str | None) -> None: + """Set ``harness_error`` (only when every run errored) and ``shadow`` on an exception a kind + raises by hand -- the counterpart of the two ``raise_agentic_failure`` arguments.""" + harness_error = labels.harness_error_record + if harness_error is not None: + exc.harness_error = harness_error + exc.shadow = labels.shadow(gate) + + +def failure_message_head( + gate: str | None, labels: RunLabels, runs_passed: int, failed_runs: Sequence[dict], runs_ungraded: int = 0 +) -> str: + """The start of a kind's failure message: the gate result, then every failing run. + + An item whose every run errored says so first. Only when every run's judge failed does it + also say no run was scored: a run whose simulated user failed was still checked, and its + failed checks follow. The gate note itself is the one the item was judged by. + """ + runs_total = len(labels.outcomes) + runs_errored = sum(1 for o in labels.outcomes if o.is_error) + note = gate_failure_note( + gate, runs_passed=runs_passed, runs_total=runs_total, runs_ungraded=runs_ungraded, runs_errored=runs_errored + ) + item = labels.item_outcome + if item is not None: + if all(o.reason == JUDGE_FAILED for o in labels.outcomes): + note = f"Harness error, no run was scored ({JUDGE_FAILED}). {note}" + else: + note = f"Harness error on every run ({item.reason}). {note}" + return failure_head(note, failed_runs, runs_total) + + +def log_eval_error_score(ctx: Any, trace_id: Any, outcome: Outcome) -> None: + """Mark one ERROR run in Langfuse: ``eval_error`` = 1, its comment the reason. + + An extra score only: the run's pass, gate, check and quality scores are written beside + it, unchanged. The marker says why the run failed. + """ + ctx.score(trace_id, name=EVAL_ERROR_SCORE, value=1, data_type="NUMERIC", comment=outcome.reason) + + +def mark_unscored_run(ctx: Any, run_idx: int, conversation_id: str, error: Outcome | None) -> None: + """Langfuse for a run that gets no verdict scores (its judge or its request failed). + + A harness ERROR (``error``, from ``RunLabels.error_label``) is attached to its experiment + item with the ``eval_error`` marker, joined or not, the way every scored run is, so an + errored run is counted wherever the item's runs are. Any other unscored run goes through + ``export_unscored``, which exports the root of a joined run only. + """ + if error is None: + ctx.export_unscored(run_idx, conversation_id) + return + with ctx.observe(ctx.trace(conversation_id), run_idx, conversation_id=conversation_id) as tid: + log_eval_error_score(ctx, tid, error) def agentic_detail( @@ -60,6 +238,8 @@ def raise_agentic_failure( timings: PhaseTimings | None = None, failed_runs: list[dict] | None = None, run_latencies_s: list[float] | None = None, + harness_error: dict[str, Any] | None = None, + shadow: dict[str, Any] | None = None, ) -> NoReturn: """Build ``exception_cls(message)`` with every common field attached, and raise it. @@ -68,6 +248,10 @@ def raise_agentic_failure( "no verdict at all" branch raises a bare ``JudgeResponseError`` instead (not a subclass) does not go through this helper for that branch. + ``harness_error`` is set only when every run errored (``RunLabels.harness_error_record``), and + ``shadow`` is the phase-2 candidate verdict (``RunLabels.shadow``); both are labels a + runner reads, neither changes the failure. + ``timings`` stays optional: only the three kinds that track per-run ``PhaseTimings`` (general_question, metric_skill, dashboard_skill) pass one -- the rest keep their own ``PhaseTimings()`` zero default, same as before this helper existed. ``failed_runs`` @@ -88,6 +272,10 @@ def raise_agentic_failure( exc.failed_runs = failed_runs if run_latencies_s is not None: exc.run_latencies_s = run_latencies_s + if harness_error is not None: + exc.harness_error = harness_error + if shadow is not None: + exc.shadow = shadow raise exc @@ -103,8 +291,13 @@ def agentic_success( timings: PhaseTimings | None = None, failed_runs: list[dict] | None = None, run_latencies_s: list[float] | None = None, + shadow: dict[str, Any] | None = None, ) -> AgenticEvalOutcome: - """The success-path mirror of ``raise_agentic_failure`` -- same fields, same shape.""" + """The success-path mirror of ``raise_agentic_failure`` -- same fields, same shape. + + No ``harness_error``: an item that passed had a run that passed, so it is never one. + ``shadow`` is recorded all the same (``RunLabels.shadow``). + """ kwargs: dict[str, Any] = { "reasoning_steps": reasoning_steps, "conversation_id": conversation_id, @@ -120,4 +313,6 @@ def agentic_success( kwargs["failed_runs"] = failed_runs if run_latencies_s is not None: kwargs["run_latencies_s"] = run_latencies_s + if shadow is not None: + kwargs["shadow"] = shadow return AgenticEvalOutcome(**kwargs) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py index 3f9d3028e..a549f6fef 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py @@ -217,9 +217,10 @@ def export_unscored(self, run_idx: int, conversation_id: str) -> None: with self.observe(self.trace(conversation_id), run_idx, conversation_id=conversation_id): pass - def score(self, trace_id: Any, *, name: str, value: Any, data_type: str) -> None: + def score(self, trace_id: Any, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: """Write one score, swallowing Langfuse failures the way ``score_safe`` always has.""" - self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type) + extra = {"comment": comment} if comment is not None else {} + self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type, **extra) def quality(self, trace_id: Any, *, strict_checks: dict, latency_sec: Any, cost_usd: Any) -> None: """Write the derived quality/value scores for one run.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index f1f532af3..ef1021942 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -15,15 +15,23 @@ from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1021,7 +1029,7 @@ def _run_once(conv_id: str) -> AlertRunResult: partial_alert_id, _, partial_tool_called = _extract_alert_call(partial.tool_call_events or []) if partial_tool_called: alert_id_to_delete = partial_alert_id - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break # `turns_used` counts attempts (set above, so a failed send still shows the # loop reached this iteration); `turns`/`steps` count completed work, so they @@ -1164,6 +1172,14 @@ def _run_detail(run: AlertRunResult) -> dict: } +def _failed_checks(run: AlertRunResult) -> list[dict]: + """The checks this run failed, by name. The arguments the run sent are in its detail. + + Every other check reads False when no alert was created, so then only that one is named. + """ + return failed_checks_from(run.eval.checks(), prerequisite="alert_created") + + def evaluate_agentic_alert_skill( host: str, token: str, @@ -1220,6 +1236,9 @@ def evaluate_agentic_alert_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.eval.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -1231,6 +1250,8 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.eval.checks() with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -1286,12 +1307,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.eval.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) raise_agentic_failure( AlertSkillAssertionError, f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " @@ -1302,6 +1324,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"attributes_correct={ev.attributes_correct}, " f"granularity_correct={ev.granularity_correct}. " f"Actual args: {best.actual_alert_arguments}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -1313,6 +1337,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py index 25f2281a0..27db1b20b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py @@ -33,15 +33,23 @@ from typing import Any from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -259,18 +267,21 @@ class AnomalyEvaluation: asserted: list[str] = field(default_factory=list) disambiguated: bool = False + @property + def strict_checks(self) -> dict[str, bool]: + """Every check strict_pass requires, by name.""" + return { + "triggered": self.triggered, + "executed": self.executed, + "success": self.success, + "turn_completed": self.turn_completed, + "metric_correct": self.metric_correct, + "granularity_correct": self.granularity_correct, + } + @property def strict_pass(self) -> bool: - return all( - [ - self.triggered, - self.executed, - self.success, - self.turn_completed, - self.metric_correct, - self.granularity_correct, - ] - ) + return all(self.strict_checks.values()) @dataclass @@ -421,7 +432,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) viz_args, execute_result = _extract_anomaly_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -535,6 +546,11 @@ def _detail(best: AnomalyRunResult) -> dict[str, Any]: ) +def _failed_checks(run: AnomalyRunResult) -> list[dict]: + """The checks this run failed, by name. What the agent sent is in the run's detail.""" + return failed_checks_from(run.evaluation.strict_checks) + + def evaluate_agentic_anomaly_detection( host: str, token: str, @@ -583,6 +599,9 @@ def evaluate_agentic_anomaly_detection( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -613,6 +632,8 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -654,13 +675,14 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_detail, + failed_checks=_failed_checks, ) runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) message = ( f"Anomaly detection assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " @@ -672,6 +694,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( AnomalyDetectionAssertionError, message, + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -684,6 +708,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 92f106340..5d227c9e8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -56,7 +56,13 @@ summarize_visualizations, ) from gooddata_eval.core.agentic._gate import log_gate_scores -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1038,7 +1044,7 @@ def run_agentic_conversation( conversation_reasoning_step_events.extend(partial.reasoning_step_events or []) reasoning_steps.extend(partial.reasoning_steps or []) response_id = partial.response_id or response_id - turn_exit = LoopExit.CHAT_ERROR + turn_exit = chat_failure_exit(exc) break final_result = chat_result total_steps += chat_result.reasoning_step_count @@ -1255,7 +1261,7 @@ def run_agentic_conversation( # A chat fault used to escape the whole run, so reaching here did mean no # error. Now that it is caught and recorded, this has to read it back -- # otherwise a turn whose chat call failed reports no_error=True. - no_error=turn_exit is not LoopExit.CHAT_ERROR, + no_error=turn_exit not in CHAT_FAILURE_EXITS, activated_skills=declared or [], active_skills=sorted(active_skills), clarification_turns_used=clarification_turns, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py index b7f2739f9..72f7e2d4e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -13,16 +13,24 @@ from gooddata_eval.core.agentic._clarification import all_questions from gooddata_eval.core.agentic._conversation_context import CONFIRMATION_REPLY, asks_in_text, classify_reply, clip from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._filters import describe_selection, same_unordered_values, selection_of from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1903,7 +1911,8 @@ class DashboardRunResult: run_latency_s: float = 0.0 # Why the loop stopped -- see LoopExit. A dashboard not produced alone cannot tell a # refusal from a run that hit max_iterations while still on track (matches - # kda_skill.py/alert_skill.py). CHAT_ERROR is reachable; SIMULATED_USER_FAILED is not -- + # kda_skill.py/alert_skill.py). The chat failure exits are reachable (see + # chat_failure_exit); SIMULATED_USER_FAILED is not -- # this loop's follow-up is built from the expectation by build_simulated_reply, with no # judge call that could fail. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @@ -2302,7 +2311,7 @@ def _execute_single_dashboard_run( all_tool_call_events.extend(partial.tool_call_events or []) all_reasoning_step_events.extend(partial.reasoning_step_events or []) steps += partial.reasoning_step_count - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -2454,10 +2463,17 @@ def _run_detail(run: DashboardRunResult) -> dict[str, Any]: **run.evaluation.diagnostics, "failures": run.evaluation.failures, "notes": run.evaluation.notes, + # Why the loop stopped -- see LoopExit. + "exit_reason": run.exit_reason.value, "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), } +def _failed_checks(run: DashboardRunResult) -> list[dict]: + """The checks this run failed, by name. What each one found is in the run's ``failures``.""" + return failed_checks_from(run.evaluation.strict_checks) + + def run_agentic_dashboard_skill( host: str, token: str, @@ -2580,6 +2596,9 @@ def evaluate_agentic_dashboard_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -2591,6 +2610,8 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.evaluation.strict_checks with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") for name, value in run.evaluation.diagnostics.items(): @@ -2640,12 +2661,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) # Two failures wear the same missing skill, and the advice differs. If the other # dashboard skill was activated, the flag that registers both is plainly on and the # model simply routed the wrong way. If neither was, the flag is off or the agent has @@ -2669,6 +2691,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Checks: {best.evaluation.strict_checks}. " f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." + (f" Notes (not scored): {notes}." if notes else ""), + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -2681,6 +2705,7 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s=[r.run_latency_s for r in summary.run_results], ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index 1b69e2175..7faaa63a7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -20,11 +20,11 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import attach_labels, failure_message_head, label_runs, mark_unscored_run from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1716,6 +1716,16 @@ class DocumentSkillAssertionError(AgenticAssertionError): """Raised when a document-skill evaluation fails.""" +def _judge_error_of(run: DocumentRunResult) -> str | None: + """The judge's own failure, for the harness label, when it is what left the run without a verdict. + + A run that already failed a fixed check is a FAIL whatever the judge would have said, so its + judge error does not make it a harness error. ``run.judge_error`` also reports a failed + takeaway request (``fill_error``), which is on the product path and stays a FAIL. + """ + return run.evaluation.judge_error if run.evaluation.ungraded else None + + def evaluate_agentic_document_skill( host: str, token: str, @@ -1795,6 +1805,9 @@ def evaluate_agentic_document_skill( document_format=document_format, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, judge_error=_judge_error_of) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -1806,7 +1819,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.ungraded: # No verdict decided this run: its other scores would publish a pass the gate # never counted. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.error_label(run_idx)) continue pt = ctx.trace(run.conversation_id) strict_checks = run.evaluation.strict_checks @@ -1866,6 +1879,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) exc_judge.timings = item_timings exc_judge.run_latencies_s = [r.run_latency_s for r in summary.run_results] + attach_labels(exc_judge, run_labels, gate) raise exc_judge runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) @@ -1896,10 +1910,14 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: } # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. - failed_runs = build_failed_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, detail=_run_detail) + failed_runs = build_failed_runs( + summary.run_results, + labels=run_labels, + detail=_run_detail, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) skill_note = "" if not best.evaluation.wire_names_current: skill_note = " gen-ai still sends the Report names, so it predates the Document rename." @@ -1923,9 +1941,11 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: exc.runs_effective = runs_effective exc.failed_runs = failed_runs exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] + attach_labels(exc, run_labels, gate) raise exc return AgenticEvalOutcome( runs_passed=runs_passed, + shadow=run_labels.shadow(gate), runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 907f0567a..7880dd021 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -7,15 +7,23 @@ from dataclasses import dataclass, field from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import judge_failed_checks from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + attach_labels, + failure_message_head, + label_runs, + mark_unscored_run, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -276,6 +284,13 @@ def _run_detail(run: GeneralQuestionResult) -> dict: } +def _failed_checks(run: GeneralQuestionResult) -> list[dict]: + """The judge's verdict on a run it failed, with its reasoning. A run it could not grade has no verdict.""" + return judge_failed_checks( + "general_question_pass", passed=run.passed, judge_error=run.judge_error, reasoning=run.reasoning + ) + + def evaluate_agentic_general_question( host: str, token: str, @@ -339,6 +354,9 @@ def evaluate_agentic_general_question( document_format=document_format, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.judge_error is None and r.passed) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -350,7 +368,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.judge_error is not None: # No verdict for this run: float(run.passed) would write a 0 the judge # never returned. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.error_label(run_idx)) continue pt = ctx.trace(run.conversation_id) with ctx.observe( @@ -410,12 +428,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # the report has to say so. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), ) - # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as - # non-passing here so it is recorded with its judge_error rather than silently dropped. + # An ungraded run has no verdict, so `r.passed` is not a claim about it: its label is not + # PASS, so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.judge_error is None and r.passed, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not summary.scored_run_results: @@ -431,13 +450,16 @@ def _write_scores(ctx: RunTraceContext) -> None: error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, run_latency_s(best.timings) ) error.timings = item_timings + attach_labels(error, run_labels, gate) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) raise_agentic_failure( GeneralQuestionAssertionError, f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -450,6 +472,7 @@ def _write_scores(ctx: RunTraceContext) -> None: timings=item_timings, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index f9e281c2d..803ec7f3c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -7,15 +7,23 @@ from dataclasses import dataclass, field from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import judge_failed_checks from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + attach_labels, + failure_message_head, + label_runs, + mark_unscored_run, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -247,6 +255,13 @@ def _run_detail(run: GuardrailResult) -> dict: } +def _failed_checks(run: GuardrailResult) -> list[dict]: + """The judge's verdict on a run it failed, with its reasoning. A run it could not grade has no verdict.""" + return judge_failed_checks( + "guardrail_pass", passed=run.passed, judge_error=run.judge_error, reasoning=run.reasoning + ) + + def evaluate_agentic_guardrail( host: str, token: str, @@ -299,6 +314,9 @@ def evaluate_agentic_guardrail( user_context=user_context, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.judge_error is None and r.passed) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -310,7 +328,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.judge_error is not None: # No verdict, so nothing truthful to publish: float(run.passed) would # write a 0 the judge never returned. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.error_label(run_idx)) continue pt = ctx.trace(run.conversation_id) with ctx.observe( @@ -368,12 +386,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # pass@K over fewer runs than --runs asked for is a weaker result. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), ) - # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as - # non-passing here so it is recorded with its judge_error rather than silently dropped. + # An ungraded run has no verdict, so `r.passed` is not a claim about it: its label is not + # PASS, so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.judge_error is None and r.passed, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not summary.scored_run_results: @@ -386,13 +405,16 @@ def _write_scores(ctx: RunTraceContext) -> None: _attach_diagnostics( error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, best.run_latency_s ) + attach_labels(error, run_labels, gate) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) raise_agentic_failure( GuardrailAssertionError, f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -404,6 +426,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 683f8d42d..e92464bf9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -11,15 +11,23 @@ import httpx from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -176,9 +184,19 @@ class KdaEvaluation: turn_completed: bool disambiguated: bool = False + @property + def strict_checks(self) -> dict[str, bool]: + """Every check strict_pass requires, by name.""" + return { + "triggered": self.triggered, + "executed": self.executed, + "success": self.success, + "turn_completed": self.turn_completed, + } + @property def strict_pass(self) -> bool: - return all([self.triggered, self.executed, self.success, self.turn_completed]) + return all(self.strict_checks.values()) @dataclass @@ -329,7 +347,7 @@ def _accumulate(result: ChatResult) -> None: if create_args is not None: turn_wall_clock_sec = partial.turn_wall_clock_sec turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break turns += 1 steps += chat_result.reasoning_step_count @@ -446,6 +464,11 @@ def _run_detail(run: KdaRunResult) -> dict: } +def _failed_checks(run: KdaRunResult) -> list[dict]: + """The checks this run failed, by name. What the agent sent is in the run's detail.""" + return failed_checks_from(run.evaluation.strict_checks) + + def evaluate_agentic_kda_skill( host: str, token: str, @@ -499,6 +522,9 @@ def evaluate_agentic_kda_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -525,6 +551,8 @@ def _write_scores(ctx: RunTraceContext) -> None: "[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, turn_wall_clock_sec ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN") @@ -585,12 +613,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) message = ( f"KDA skill assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, " @@ -601,6 +630,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( KdaSkillAssertionError, message, + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -612,6 +643,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index f662b205b..17d19182e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -13,15 +13,23 @@ from gooddata_sdk import GoodDataSdk from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -352,7 +360,7 @@ def _execute_single_metric_run( for metric_id in _extract_created_metric_ids(partial.tool_call_events or []): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -562,6 +570,9 @@ def evaluate_agentic_metric_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.metric_created and r.maql_correct) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -577,6 +588,8 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=run.conversation_id, output={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, ) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -631,6 +644,25 @@ def _run_detail(run: MetricRunResult) -> dict: **timeline_detail(run.tool_call_events, run.reasoning_step_events), } + def _failed_checks(run: MetricRunResult) -> list[dict]: + """The checks this run failed; the MAQL pair is the whole diagnosis for this kind. + + With no metric, ``maql_correct`` reads False only because there is no MAQL to compare, + so ``metric_created`` alone is named. + """ + checks: list[FailedCheck] = [] + if not run.metric_created: + checks.append(FailedCheck("metric_created", expected=True, actual=False)) + elif not run.maql_correct: + checks.append( + FailedCheck( + "maql_correct", + expected=[c.get("maql", "") for c in expected_outputs_list], + actual=run.actual_maql, + ) + ) + return failed_check_records(checks) + detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, @@ -648,12 +680,13 @@ def _run_detail(run: MetricRunResult) -> dict: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.metric_created and r.maql_correct, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) raise_agentic_failure( MetricSkillAssertionError, @@ -661,6 +694,8 @@ def _run_detail(run: MetricRunResult) -> dict: f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " f"Actual MAQL: {best.actual_maql}.", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -673,6 +708,7 @@ def _run_detail(run: MetricRunResult) -> dict: run_latencies_s=run_latencies_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index b06821452..87119eb89 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -12,15 +12,24 @@ from dataclasses import dataclass, field, replace from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -232,21 +241,21 @@ def _execute_single_run( current_result = client.send_message(conversation_id, question) except ChatError as exc: # The opening request, so there is no partial conversation to evaluate: fall through - # to the empty-result path with CHAT_ERROR recorded. Raising here would discard every + # to the empty-result path with its exit recorded. Raising here would discard every # K-run already completed along with any exit_reason. print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}") current_result = getattr(exc, "partial_result", None) or ChatResult() - # The loop that collects events does not run after CHAT_ERROR, so a partial result's + # The loop that collects events does not run after a failed opening request, so a partial result's # chart executions are added here. all_tool_call_events.extend(current_result.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) # Counted here, not at the top of the loop: this request is sent unconditionally, so # with max_iterations=0 the loop body never runs and total_turns would report 0 turns # for a conversation the agent did receive. The loop's own increment is skipped on its # first pass to compensate, keeping total_turns == number of send_message calls. total_turns += 1 - for iteration in range(max_iterations if exit_reason is not LoopExit.CHAT_ERROR else 0): + for iteration in range(max_iterations if exit_reason not in CHAT_FAILURE_EXITS else 0): if iteration: total_turns += 1 total_steps += current_result.reasoning_step_count @@ -297,7 +306,7 @@ def _execute_single_run( # The loop adds a turn's events at its next pass, which a break skips. The # partial result is scored, so its chart executions count too. all_tool_call_events.extend(partial.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break skill_activated = _check_visualization_skill_activated(all_tool_call_events) @@ -316,7 +325,7 @@ def _execute_single_run( actual_output, all_tool_call_events, answer_text, - interrupted=exit_reason is LoopExit.CHAT_ERROR, + interrupted=exit_reason in CHAT_FAILURE_EXITS, ) return RunResult( @@ -445,6 +454,81 @@ def _run_detail(run: RunResult) -> dict: } +def _failed_checks(run: RunResult) -> list[dict]: + """The checks this run failed, by their ``detail`` names, with expected/actual where known. + + Follows ``EvaluationResult.strict_pass`` term by term, so the list is empty exactly when + the run passed. + """ + ev = run.eval_result + checks: list[FailedCheck] = [] + if ev.scored_without_chart: + if ev.visualization_created: + actual_type = run.actual_output.type if run.actual_output else None + checks.append(FailedCheck("no_chart", expected="no chart", actual=actual_type)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + return failed_check_records(checks) + if not ev.visualization_created: + # Every chart check reads False without a chart; listing them would name content + # failures for work the agent never did. The answer checks still read the text. + checks.append(FailedCheck("visualization_created", expected=True, actual=False)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + if ev.format_correct is False: + checks.append(FailedCheck("format_correct", actual=ev.format_problems)) + return failed_check_records(checks) + if not ev.cross_ref_valid: + checks.append(FailedCheck("cross_ref_valid", actual=ev.cross_ref_errors)) + if not ev.metrics_correct: + if ev.expected_metric_uris == ev.actual_metric_uris and ev.metric_layout_problems: + # The metric set matched; the order or axis did not, and only these say which. + checks.append(FailedCheck("metrics_correct", part="layout", actual=ev.metric_layout_problems)) + else: + checks.append( + FailedCheck( + "metrics_correct", expected=sorted(ev.expected_metric_uris), actual=sorted(ev.actual_metric_uris) + ) + ) + if not ev.dimensions_correct: + checks.append( + FailedCheck("dimensions_correct", expected=sorted(ev.expected_dim_uris), actual=sorted(ev.actual_dim_uris)) + ) + if not ev.filters_correct: + differing = [ + FailedCheck( + "filters_correct", + part=category, + expected=ev.expected_filters.get(category, []), + actual=ev.actual_filters.get(category, []), + ) + for category in sorted(set(ev.expected_filters) | set(ev.actual_filters)) + if ev.expected_filters.get(category, []) != ev.actual_filters.get(category, []) + ] + checks.extend(differing or [FailedCheck("filters_correct")]) + if not ev.sorts_correct: + checks.append(FailedCheck("sorts_correct", expected=ev.expected_sorts, actual=ev.actual_sorts)) + if not ev.viz_type_hard: + checks.append( + FailedCheck( + "viz_type_hard", + expected=run.best_expected.type if run.best_expected else None, + actual=run.actual_output.type if run.actual_output else None, + ) + ) + if ev.requires_execution and not ev.executed: + checks.append(FailedCheck("executed", expected=True, actual=False)) + if ev.buckets_correct is False: + checks.append(FailedCheck("buckets_correct", expected=ev.expected_buckets, actual=ev.actual_buckets)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + if ev.format_correct is False: + checks.append(FailedCheck("format_correct", actual=ev.format_problems)) + if ev.part_present is False: + checks.append(FailedCheck("part_present", expected=True, actual=False)) + return failed_check_records(checks) + + def _strict_checks(ev: EvaluationResult) -> dict[str, bool]: """The scores one run is gated on, by their Langfuse names. @@ -579,6 +663,9 @@ def evaluate_agentic_visualization( options=options, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.eval_result.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -592,6 +679,8 @@ def _write_scores(ctx: RunTraceContext) -> None: ev = run.eval_result strict_checks = _strict_checks(ev) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): # Scored below on every item, gating or not. if score_name != "assertion-vis-executed": @@ -662,12 +751,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # until now only the winning run's survived. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.eval_result.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) n = len(expected_outputs) candidate_note = f" (closest of {n} candidates)" if n > 1 else "" cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else "" @@ -711,6 +801,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"{_options_lines(ev, best.answer_text)}" f" Stated Value Matches : {ev.stated_value_matches} (reported only)\n" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -722,6 +814,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py index de124ceed..c6281b630 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py @@ -27,15 +27,23 @@ from typing import Any from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -184,20 +192,23 @@ class WhatIfEvaluation: asserted: list[str] = field(default_factory=list) disambiguated: bool = False + @property + def strict_checks(self) -> dict[str, bool]: + """Every check strict_pass requires, by name.""" + return { + "triggered": self.triggered, + "executed": self.executed, + "success": self.success, + "turn_completed": self.turn_completed, + "metric_correct": self.metric_correct, + "maql_correct": self.maql_correct, + "scenario_count_correct": self.scenario_count_correct, + "baseline_correct": self.baseline_correct, + } + @property def strict_pass(self) -> bool: - return all( - [ - self.triggered, - self.executed, - self.success, - self.turn_completed, - self.metric_correct, - self.maql_correct, - self.scenario_count_correct, - self.baseline_correct, - ] - ) + return all(self.strict_checks.values()) @dataclass @@ -378,7 +389,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) create_args, execute_result = _extract_what_if_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -496,6 +507,11 @@ def _detail(best: WhatIfRunResult) -> dict[str, Any]: ) +def _failed_checks(run: WhatIfRunResult) -> list[dict]: + """The checks this run failed, by name. What the agent sent is in the run's detail.""" + return failed_checks_from(run.evaluation.strict_checks) + + def evaluate_agentic_what_if( host: str, token: str, @@ -544,6 +560,9 @@ def evaluate_agentic_what_if( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -576,6 +595,8 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -618,13 +639,14 @@ def _write_scores(ctx: RunTraceContext) -> None: # run is reported with exactly the keys the winning one is. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_detail, + failed_checks=_failed_checks, ) runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) message = ( f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " @@ -637,6 +659,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( WhatIfAssertionError, message, + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -649,6 +673,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index ce7ead5c1..78a7388aa 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -340,6 +340,10 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult: return result +def _budget_exceeded(budget: float, scope: str) -> TurnTimeoutError: + return TurnTimeoutError(f"agent exceeded the {budget:.0f}s {scope} budget") + + def _until_deadline( lines: Iterable[str], deadline: float | None, budget: float = 0.0, scope: str = "turn" ) -> Iterable[str]: @@ -351,9 +355,21 @@ def _until_deadline( if deadline is None: yield from lines return - for line in lines: + it = iter(lines) + while True: + try: + line = next(it) + except StopIteration: + return + except httpx.ReadTimeout as exc: + # The read timeout is lowered to the caps, so an agent that goes silent past its + # budget ends here rather than at the check below. Past the deadline it is the same + # timeout; before it, a plain read timeout stays a transport failure. + if time.monotonic() > deadline: + raise _budget_exceeded(budget, scope) from exc + raise if time.monotonic() > deadline: - raise TurnTimeoutError(f"agent exceeded the {budget:.0f}s {scope} budget") + raise _budget_exceeded(budget, scope) yield line @@ -591,16 +607,27 @@ def _do() -> ChatResult: # attempts, but the entire duration of any earlier failed attempt. t0 = time.monotonic() deadline, budget, scope = self._deadline(t0) - with self._client.stream("POST", url, json=body, headers=headers) as resp: - resp.raise_for_status() - try: - result = parse_sse_lines(_until_deadline(resp.iter_lines(), deadline, budget, scope)) - except ChatError as exc: - if exc.partial_result is not None: - exc.partial_result.turn_wall_clock_sec = time.monotonic() - t0 - raise - result.turn_wall_clock_sec = time.monotonic() - t0 - return result + try: + with self._client.stream("POST", url, json=body, headers=headers) as resp: + resp.raise_for_status() + try: + result = parse_sse_lines(_until_deadline(resp.iter_lines(), deadline, budget, scope)) + except ChatError as exc: + if exc.partial_result is not None: + exc.partial_result.turn_wall_clock_sec = time.monotonic() - t0 + raise + result.turn_wall_clock_sec = time.monotonic() - t0 + return result + except httpx.ReadTimeout as exc: + # A read timeout in the stream body is a ChatError by now, so this one came while + # the response headers were awaited. An agent silent past its budget can end here + # too when the headers are held back until the first event. Such a turn has no + # partial result to carry its time, so it is recorded here: it is one of the slowest. + now = time.monotonic() + if deadline is not None and now > deadline: + note_trace_latency(now - t0) + raise _budget_exceeded(budget, scope) from exc + raise try: result = retry_transient(_do, is_retryable=_is_retryable_exc) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py index 6bb4b9e62..2c3230f83 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py @@ -63,6 +63,10 @@ class JudgeResponseError(RuntimeError): failed_runs: list[dict] run_latencies_s: list[float] best_run_latency_s: float | None + # The item's harness label and the phase-2 candidate verdict (``core/outcome.py``), as on + # AgenticAssertionError. + harness_error: dict + shadow: dict def _message_content(response: Any) -> str | None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 094cee08e..391267b78 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -25,9 +25,14 @@ class LoopExit(str, Enum): per-field check as False: specific-sounding content failures for work the agent was never given the chance to do. - This does not change any verdict -- an exhausted run still fails. It makes the three + This does not change any verdict -- an exhausted run still fails. It makes these cases countable, so "is the turn budget too tight" becomes a question the data can answer rather than one that has to be argued. + + Which members label a run ERROR (the harness failed) rather than FAIL is decided in one + place, ``core/outcome.py``. The label decides no verdict. Only SIMULATED_USER_FAILED is an + ERROR: CHAT_ERROR, AGENT_TIMEOUT and TURN_INCOMPLETE are on the product path, and a + customer hits them too. """ SUCCESS = "success" @@ -48,16 +53,27 @@ class LoopExit(str, Enum): SIMULATED_USER_FAILED = "simulated_user_failed" """The harness's own simulated-user model failed to produce a follow-up. A harness-side - fault, not an agent one -- reporting should treat it as an error, not a scored failure.""" + fault, not an agent one, so the run is labelled ERROR (``core/outcome.py``).""" CHAT_ERROR = "chat_error" - """The chat call itself raised mid-conversation. Like SIMULATED_USER_FAILED this is - infrastructure rather than agent capability, but it comes from the GoodData side.""" + """The chat call itself raised mid-conversation and was not one of the agent's own + failures below: a gen-ai error event, a broken stream, or a 5xx still failing after its + retries. It is on the product path -- a customer would see it too -- so the run is a + FAIL, not a harness ERROR.""" + + AGENT_TIMEOUT = "agent_timeout" + """The agent exceeded the per-turn or per-item wall-clock budget (``TurnTimeoutError``). + The agent's own failure to answer in time, so the run is a FAIL.""" + + TURN_INCOMPLETE = "turn_incomplete" + """The agent ended its turn without a final answer, e.g. at its own iteration limit + (``TurnIncompleteError``). The agent's own failure, so the run is a FAIL.""" NOT_RUN = "not_run" """The loop never started -- e.g. an agentic_conversation turn whose $ref to an earlier turn's output could not be resolved, so the turn was skipped before any message was - sent. Distinct from every other member, which all imply at least one turn happened.""" + sent. Distinct from every other member, which all imply at least one turn happened. A + FAIL: its cause is usually the earlier turn's failure, which the item already reports.""" class AacQueryField(BaseModel): @@ -441,6 +457,12 @@ class AgenticAssertionError(AssertionError): failed_runs: list[dict] run_latencies_s: list[float] best_run_latency_s: float | None + harness_error: dict + """``{"status": "error", "reason": ...}``, set only when every run of the item errored + (``core/outcome.py``): the item is a harness error, not an agent failure. A runner labels + it; it stays a failure.""" + shadow: dict + """The phase-2 candidate verdict (``core.outcome.shadow_verdict``). Recorded, never used.""" class AgenticEvalOutcome(BaseModel): @@ -483,6 +505,10 @@ class AgenticEvalOutcome(BaseModel): # not in a loop the runner itself can time. None for a kind that has not yet been # wired to measure it (see AgenticAssertionError.best_run_latency_s, same default). best_run_latency_s: float | None = None + # The phase-2 candidate verdict (``core.outcome.shadow_verdict``), as on + # AgenticAssertionError. Recorded on a passing item too: under pass@K an item can pass with + # most of its runs errored, which the candidate rule would not call a pass. Never used. + shadow: dict | None = None class SummaryInput(BaseModel): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py new file mode 100644 index 000000000..0d9d0df95 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py @@ -0,0 +1,159 @@ +# (C) 2026 GoodData Corporation +"""One label per run: PASS, FAIL, or ERROR when the test harness, not the agent, failed. + +A run can end in a way that is the harness's fault: the simulated user's own model call +fails, or the judge cannot grade the answer. Scored as usual, such a run reads as an agent +failure. ``classify_run`` names the difference, in one place for every kind. + +The labels decide no verdict. A run labelled ERROR still counts as a run that did not pass, +so a gate, a pass rate or a trend reads the same with or without them. The labels only tell +a reader why a run did not pass. ``shadow_verdict`` records what a candidate rule that leaves +ERROR runs out of the gate would have given, so that rule can be judged on real nights +before anyone adopts it. + +The rule is fail-closed. A run is ERROR only for a cause on ``_ERROR_EXITS`` or a judge +failure that left the run without a verdict, causes a customer would not hit. That is not +proof the agent was blameless: the simulated user is also asked after an answer that was +already final, so an ERROR run still lists the checks it failed. Anything else, including a +chat API error and an unknown exit, is FAIL. +""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from gooddata_eval.core.config import normalize_gate +from gooddata_eval.core.models import LoopExit + +__all__ = [ + "EVAL_ERROR_SCORE", + "JUDGE_FAILED", + "Outcome", + "OutcomeStatus", + "SHADOW_MIN_SCORED_SHARE", + "UNKNOWN_REASON", + "classify_run", + "item_outcome", + "shadow_verdict", +] + + +class OutcomeStatus(str, Enum): + """What one run, or one item, amounts to.""" + + PASS = "pass" + FAIL = "fail" + ERROR = "error" + """The test harness failed, so the run says nothing about the agent.""" + + +@dataclass(frozen=True) +class Outcome: + """A status and a machine-readable reason, e.g. ``simulated_user_failed``.""" + + status: OutcomeStatus + reason: str | None = None + + @classmethod + def passed(cls) -> Outcome: + return cls(OutcomeStatus.PASS) + + @classmethod + def failed(cls, reason: str | None = None) -> Outcome: + return cls(OutcomeStatus.FAIL, reason) + + @classmethod + def error(cls, reason: str) -> Outcome: + return cls(OutcomeStatus.ERROR, reason) + + @property + def is_error(self) -> bool: + return self.status is OutcomeStatus.ERROR + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into ``failed_runs[].outcome`` and onto the exception.""" + return {"status": self.status.value, "reason": self.reason} + + +EVAL_ERROR_SCORE = "eval_error" +"""The extra Langfuse score (NUMERIC, 1) an ERROR run gets, its comment the reason. The +run's other scores are written as always.""" + +JUDGE_FAILED = "judge_failed" +"""The reason of a run the judge or scorer could not grade.""" + +UNKNOWN_REASON = "unknown" +"""The reason shown for an ERROR that carries none.""" + +SHADOW_MIN_SCORED_SHARE = 0.5 +"""The share of an item's K runs the candidate rule needs scored, rounded up, before it judges +the item: ceil(K/2). A placeholder until real nights say what it should be.""" + +# The only loop exits that make a run an ERROR. A chat API error stays a FAIL: it is on the +# product path, and a customer would hit it too. +_ERROR_EXITS: frozenset[LoopExit] = frozenset({LoopExit.SIMULATED_USER_FAILED}) + + +def classify_run(exit_reason: LoopExit | str | None, *, passed: bool, judge_error: str | None = None) -> Outcome: + """The label of one run, from whether it passed, why its loop stopped and its judge. + + A run that passed is PASS whatever its exit: its checks held on what the agent produced. + A run whose judge failed is ERROR (``judge_failed``). The caller passes ``judge_error`` only + when the judge's verdict was the one still open: a run that already failed another check is + a FAIL whatever the judge would have said. A run is ERROR too when its exit is on + ``_ERROR_EXITS``. Every other run is FAIL, its reason the exit (or None for a kind whose + loop records none). An exit string that is not a ``LoopExit`` value is kept as the reason + of a FAIL rather than raising: an unknown cause is never an error. + """ + if passed: + return Outcome.passed() + if judge_error is not None: + return Outcome.error(JUDGE_FAILED) + loop_exit: LoopExit | None + if isinstance(exit_reason, LoopExit) or exit_reason is None: + loop_exit = exit_reason + else: + try: + loop_exit = LoopExit(exit_reason) + except ValueError: + return Outcome.failed(exit_reason) + if loop_exit in _ERROR_EXITS: + return Outcome.error(loop_exit.value) + return Outcome.failed(loop_exit.value if loop_exit is not None else None) + + +def item_outcome(outcomes: Sequence[Outcome]) -> Outcome | None: + """ERROR when every run of the item is ERROR, with the most frequent reason; else None. + + Of reasons equally frequent, the alphabetically first is named, so the label is stable. + + None means the item keeps the verdict its gate gave it. An item with some ERROR runs is + not an ERROR: its other runs were scored, and its message names the ERROR runs. + """ + if not outcomes or not all(o.is_error for o in outcomes): + return None + reasons = Counter(o.reason or UNKNOWN_REASON for o in outcomes) + return Outcome.error(max(sorted(reasons), key=lambda reason: reasons[reason])) + + +def shadow_verdict(outcomes: Sequence[Outcome], gate: str | None) -> dict[str, Any]: + """What the candidate phase-2 rule would give this item. Recorded, never used. + + The rule leaves ERROR runs out of K. With fewer scored runs left than + ``SHADOW_MIN_SCORED_SHARE`` of K the item is ``error``; otherwise ``gate`` is applied to + the scored runs. The verdict is an ``OutcomeStatus`` value. + """ + scored = [o for o in outcomes if not o.is_error] + passes = sum(1 for o in scored if o.status is OutcomeStatus.PASS) + if not scored or len(scored) < math.ceil(len(outcomes) * SHADOW_MIN_SCORED_SHARE): + verdict = OutcomeStatus.ERROR + elif normalize_gate(gate) == "power": + verdict = OutcomeStatus.PASS if passes == len(scored) else OutcomeStatus.FAIL + else: + verdict = OutcomeStatus.PASS if passes else OutcomeStatus.FAIL + return {"verdict": verdict.value, "runs_scored": len(scored), "runs_total": len(outcomes)} diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index 71df0abf8..dda6bb711 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -72,6 +72,9 @@ def _build_run_dict(report: EvalReport) -> dict: "runs_passed": item.runs_passed, "runs_ungraded": item.runs_ungraded, "pass_power_k": item.pass_power_k, + # Labels only (core/outcome.py): neither changes gate_passed. + "harness_error": item.harness_error, + "shadow": item.shadow, "best_run_latency_s": ( round(item.best_run_latency_s, 3) if item.best_run_latency_s is not None else None ), diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 2d95d529a..c93488faa 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -98,6 +98,11 @@ class ItemReport: # a failure: excluded from runs_passed and from pass@K, and pass^K is False while any # exist. Counted in `runs`, because they cost real agent time. runs_ungraded: int = 0 + # The agentic kinds' labels (``core/outcome.py``): ``harness_error`` is set only when every + # run was a harness error, ``shadow`` is the phase-2 candidate verdict. Neither decides + # anything here; they are reported for reading. + harness_error: dict | None = None + shadow: dict | None = None # What the item actually ran, when the kind knows better than the requested K. # agentic_conversation drives its fixture exactly once whatever --runs says, so # trusting K there reports four runs that never happened. diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 09cfab387..9513102e3 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -26,7 +26,7 @@ render_turn_for_simulated_user, run_agentic_alert_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _DATE_FILTER = { @@ -1932,3 +1932,21 @@ def test_an_alert_created_before_the_stream_broke_is_still_cleaned_up(): assert mock_delete.call_count == 1 assert mock_delete.call_args.args[-1] == "alert-1" + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + ], +) +def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = exc + + detail = _run_alert(mock_client, max_iterations=3) + + assert detail["exit_reason"] == exit_value + assert detail["alert_created"] is False diff --git a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py index 5c1e25d8b..4bdfe156b 100644 --- a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py +++ b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py @@ -16,6 +16,7 @@ evaluate_agentic_anomaly_detection, run_agentic_anomaly_detection, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.anomaly_detection" @@ -431,7 +432,7 @@ def trace(self, _conversation_id): def observe(self, _trace, _run_idx, *, conversation_id=None): yield "trace-id" - def score(self, _tid, *, name, value, data_type): + def score(self, _tid, *, name, value, data_type, comment=None): self.scores[name] = value def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): @@ -539,3 +540,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index ca0f2dc2d..f08b0d63f 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -53,7 +53,7 @@ run_agentic_conversation, ) from gooddata_eval.core.chat.render import render_answer_text -from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.chat.sse_client import ChatError as ChatError_ from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent @@ -1209,6 +1209,32 @@ def test_a_partial_result_joins_the_shared_timeline_like_any_other_turn(): assert result.total_steps == 5 +def test_an_agent_timeout_ends_its_turn_with_its_own_exit_and_still_reads_as_an_error(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [ + _metric_turn_result([_skills_tc("metric"), _create_metric_tc("m1")]), + TurnTimeoutError("agent exceeded the 60s turn budget"), + ] + + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + ): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=_two_metric_turn_fixture(), + ) + + assert result.turn_results[0].exit_reason is LoopExit.SUCCESS + assert result.turn_results[1].exit_reason is LoopExit.AGENT_TIMEOUT + # The turn's no_error reads exactly as it did when a timeout was recorded as CHAT_ERROR. + assert result.turn_results[1].no_error is False + assert result.turn_results[1].output_present is False + + def test_a_non_chat_exception_still_propagates(): """Only chat faults are absorbed. A programming error must not be relabelled CHAT_ERROR.""" mock_client = MagicMock() diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py index 9b6075b9f..d053be358 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -20,7 +20,7 @@ evaluate_dashboard_response, run_agentic_dashboard_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent # Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of @@ -2947,6 +2947,22 @@ def test_a_caller_supplied_conversation_is_not_deleted(self): client.create_conversation.assert_not_called() client.delete_conversation.assert_not_called() + @pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], + ) + def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(self, exc, loop_exit) -> None: + client = MagicMock() + client.send_message.side_effect = exc + + summary = _run_with(client, _DC05_EXPECTED) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + def test_a_chat_error_is_recorded_rather_than_left_to_escape(self) -> None: """It used to escape run_agentic_dashboard_skill, taking every completed K-run with it: no DashboardRunResult, no exit_reason, just a bare error on the item. Recorded diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index e1a0c0f2d..75f2a4e10 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -1724,15 +1724,22 @@ def test_the_last_document_part_of_a_turn_is_the_one_scored() -> None: class _FakeCtx: - """Records what the deferred Langfuse block writes, without a Langfuse.""" + """Records what the deferred Langfuse block writes, without a Langfuse. + + ``scores`` and ``comments`` hold every score whatever its run; ``run_scores`` and + ``run_comments`` hold them per conversation, for a test that cares which run got a score. + """ def __init__(self) -> None: self.run_metadata: dict[str, Any] = {} self.scores: dict[str, float] = {} + self.run_scores: dict[str, dict[str, float]] = {} + self.run_comments: dict[str, dict[str, str]] = {} self.score_types: dict[str, str] = {} self.quality_checks: dict[str, bool] = {} self.observed: list[str] = [] self.unscored: list[str] = [] + self.comments: dict[str, str] = {} def export_unscored(self, _run_idx: int, conversation_id: str) -> None: self.unscored.append(conversation_id) @@ -1741,13 +1748,19 @@ def trace(self, _conversation_id: str) -> None: return None @contextmanager - def observe(self, _trace: None, _run_idx: int, *, conversation_id: str, output: dict) -> Iterator[str]: + def observe( + self, _trace: None, _run_idx: int, *, conversation_id: str, output: dict | None = None + ) -> Iterator[str]: self.observed.append(conversation_id) - yield "trace-id" + yield conversation_id - def score(self, _tid: str, *, name: str, value: float, data_type: str) -> None: + def score(self, tid: str, *, name: str, value: float, data_type: str, comment: str | None = None) -> None: self.scores[name] = value + self.run_scores.setdefault(tid, {})[name] = value self.score_types[name] = data_type + if comment is not None: + self.comments[name] = comment + self.run_comments.setdefault(tid, {})[name] = comment def quality( self, _tid: str, *, strict_checks: dict[str, bool], latency_sec: float | None, cost_usd: float | None @@ -1991,12 +2004,16 @@ def test_an_ungraded_run_keeps_pass_at_k_but_not_pass_power_k(monkeypatch: pytes assert summary.best.judge_error is None -def test_an_ungraded_run_writes_no_scores(monkeypatch: pytest.MonkeyPatch) -> None: +def test_a_run_the_judge_could_not_grade_gets_only_the_harness_marker(monkeypatch: pytest.MonkeyPatch) -> None: ctx = _scored(monkeypatch, {"narrative": _NARRATIVE}, _narrative_turns(), k=2, judge=_SequenceJudge()) - assert ctx.observed == ["conv-2"] - assert ctx.unscored == ["conv-1"] - assert ctx.scores["document_narrative_judged"] == 1.0 - assert ctx.scores["document_summaries_present"] == 1.0 + # conv-1 has no verdict: its root carries eval_error alone; conv-2 gets its usual scores. + assert ctx.observed == ["conv-1", "conv-2"] + assert ctx.unscored == [] + assert ctx.run_scores["conv-1"] == {"eval_error": 1} + assert ctx.run_comments["conv-1"] == {"eval_error": "judge_failed"} + assert "eval_error" not in ctx.run_scores["conv-2"] + assert ctx.run_scores["conv-2"]["document_narrative_judged"] == 1.0 + assert ctx.run_scores["conv-2"]["document_summaries_present"] == 1.0 def _failing_narrative_turn() -> ChatResult: @@ -2009,8 +2026,13 @@ def test_a_run_that_failed_a_fixed_check_is_a_failure_even_when_the_judge_errors expected = {"narrative": _NARRATIVE, "visualizations": [{"id": _RETURNS_BY_CATEGORY, "title": "Returns"}]} _install_client(monkeypatch, _ScriptedChatClient([_failing_narrative_turn()])) judge = _FakeJudge(error=JudgeResponseError("empty body")) - with pytest.raises(DocumentSkillAssertionError, match="does not show chart 'returns_by_category'"): + with pytest.raises(DocumentSkillAssertionError, match="does not show chart 'returns_by_category'") as exc_info: evaluate_agentic_document_skill("https://h", "tok", "ws", "q", expected, judge=judge) + # Labelled a FAIL too: the judge breaking did not decide this run, the missing chart did. + exc = exc_info.value + assert [r["outcome"]["status"] for r in exc.failed_runs] == ["fail"] + assert not hasattr(exc, "harness_error") + assert "Harness error" not in str(exc) def test_a_failed_run_with_a_judge_error_is_published(monkeypatch: pytest.MonkeyPatch) -> None: @@ -2019,6 +2041,8 @@ def test_a_failed_run_with_a_judge_error_is_published(monkeypatch: pytest.Monkey assert ctx.observed == ["conv-1"] assert ctx.scores["document_charts_matched"] == 0.0 assert "document_narrative_judged" not in ctx.scores + # A FAIL, not a harness error: no marker. + assert "eval_error" not in ctx.run_scores["conv-1"] def test_only_content_pages_count_for_summaries() -> None: diff --git a/packages/gooddata-eval/tests/test_agentic_failure_message.py b/packages/gooddata-eval/tests/test_agentic_failure_message.py new file mode 100644 index 000000000..04b167bf0 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_failure_message.py @@ -0,0 +1,522 @@ +# (C) 2026 GoodData Corporation +"""The failing-runs head of an agentic failure message, and the per-run records it reads.""" + +import importlib +import inspect +from collections.abc import Callable +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import metric_skill as metric_skill_mod +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import ( + FailedCheck, + failed_check_records, + failing_runs_note, + failure_head, +) +from gooddata_eval.core.agentic._outcome import RunLabels +from gooddata_eval.core.agentic.alert_skill import AlertEvaluation +from gooddata_eval.core.agentic.alert_skill import _failed_checks as alert_failed_checks +from gooddata_eval.core.agentic.anomaly_detection import AnomalyEvaluation +from gooddata_eval.core.agentic.anomaly_detection import _failed_checks as anomaly_failed_checks +from gooddata_eval.core.agentic.general_question import _failed_checks as general_question_failed_checks +from gooddata_eval.core.agentic.guardrail import _failed_checks as guardrail_failed_checks +from gooddata_eval.core.agentic.kda_skill import KdaEvaluation +from gooddata_eval.core.agentic.kda_skill import _failed_checks as kda_failed_checks +from gooddata_eval.core.agentic.metric_skill import MetricSkillAssertionError, evaluate_agentic_metric_skill +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.agentic.what_if import WhatIfEvaluation +from gooddata_eval.core.agentic.what_if import _failed_checks as what_if_failed_checks +from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit +from gooddata_eval.core.outcome import Outcome, classify_run + +# ── the renderer ───────────────────────────────────────────────────────────── + + +def test_a_failed_check_records_only_what_the_kind_knows() -> None: + assert FailedCheck("metric_correct").as_record() == {"name": "metric_correct"} + assert FailedCheck("filters_correct", part="date", expected=[], actual=["MONTH -11..0"]).as_record() == { + "name": "filters_correct", + "part": "date", + "expected": [], + "actual": ["MONTH -11..0"], + } + # None is a real expected value, not "unknown". + assert FailedCheck("x", expected=None).as_record() == {"name": "x", "expected": None} + + +def test_the_note_lists_each_failing_run_with_its_exit_and_failing_checks_only() -> None: + failed_runs = [ + { + "run_index": 2, + "exit_reason": "success", + "failed_checks": failed_check_records( + [FailedCheck("filters_correct", part="date", expected=[], actual="MONTH -11..0")] + ), + }, + {"run_index": 3, "exit_reason": "budget_exhausted", "failed_checks": [{"name": "metric_correct"}]}, + ] + + note = failing_runs_note(failed_runs, 3) + + assert note == ( + "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual MONTH -11..0]; " + "run 3/3 FAIL (loop exit: budget_exhausted): metric_correct" + ) + + +def test_the_head_starts_with_the_gate_note() -> None: + gate_note = "Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure." + failed_runs = [{"run_index": 2, "exit_reason": "success", "failed_checks": [{"name": "metric_correct"}]}] + + head = failure_head(gate_note, failed_runs, 3) + + assert head.startswith(gate_note) + assert head.endswith("Failing runs: run 2/3 FAIL (loop exit: success): metric_correct.") + + +def test_the_head_is_the_gate_note_alone_without_failing_runs() -> None: + assert failure_head("Gate pass@2 failed: 0/2 runs passed.", [], 2) == "Gate pass@2 failed: 0/2 runs passed." + + +def test_a_run_without_recorded_checks_says_so_and_a_kind_without_the_key_says_nothing() -> None: + assert failing_runs_note([{"run_index": 1, "exit_reason": None, "failed_checks": []}], 1) == ( + "run 1/1 FAIL: no failing check recorded" + ) + assert failing_runs_note([{"run_index": 1, "exit_reason": "agent_silent"}], 1) == ( + "run 1/1 FAIL (loop exit: agent_silent)" + ) + + +def test_an_errored_run_is_labelled_with_its_reason_and_still_lists_its_checks() -> None: + record = { + "run_index": 1, + "exit_reason": "simulated_user_failed", + "outcome": {"status": "error", "reason": "simulated_user_failed"}, + "failed_checks": [{"name": "metric_created"}], + } + assert failing_runs_note([record], 3) == "run 1/3 ERROR (simulated_user_failed): metric_created" + + +def test_an_errored_run_without_failed_checks_is_its_label_alone() -> None: + record = {"run_index": 2, "outcome": {"status": "error", "reason": "judge_failed"}, "failed_checks": []} + assert failing_runs_note([record], 3) == "run 2/3 ERROR (judge_failed)" + + +def _long_run(run_index: int) -> dict: + checks = [{"name": f"check_{i}", "expected": "e" * 150, "actual": "a" * 150} for i in range(4)] + return {"run_index": run_index, "exit_reason": "budget_exhausted", "failed_checks": checks} + + +def test_failing_runs_past_the_cap_are_counted_not_rendered() -> None: + note = failing_runs_note([_long_run(i) for i in range(1, 6)], 5) + assert len(note) < 2100 + assert note.startswith("run 1/5 FAIL (loop exit: budget_exhausted): check_0[") + assert note.endswith("; 4 more failing run(s) in failed_runs") + + +def test_the_first_failing_run_is_shown_even_past_the_cap() -> None: + record = {"run_index": 1, "failed_checks": [{"name": f"check_{i}", "actual": "a" * 150} for i in range(30)]} + note = failing_runs_note([record], 1) + assert note.startswith("run 1/1 FAIL: check_0[") + assert note.endswith("...") + assert len(note) == 2000 + + +def test_a_long_value_is_cut_to_one_short_line() -> None: + record = {"run_index": 1, "failed_checks": [{"name": "maql_correct", "actual": "SELECT\n" + "x" * 500}]} + note = failing_runs_note([record], 1) + assert "\n" not in note + assert note.endswith("...]") + assert len(note) < 250 + + +# ── the records ────────────────────────────────────────────────────────────── + + +@dataclass +class _Run: + ok: bool + exit_reason: LoopExit = LoopExit.SUCCESS + conversation_id: str = "c" + tool_call_events: list = field(default_factory=list) + + +def test_every_failing_record_carries_its_exit_reason_and_failed_checks() -> None: + runs = [_Run(True), _Run(False, LoopExit.BUDGET_EXHAUSTED)] + + records = build_failed_runs( + runs, + passed=lambda r: r.ok, + detail=lambda r: {}, + failed_checks=lambda r: [{"name": "metric_correct"}], + ) + + assert len(records) == 1 + assert records[0]["run_index"] == 2 + assert records[0]["exit_reason"] == "budget_exhausted" + assert records[0]["failed_checks"] == [{"name": "metric_correct"}] + + +def test_a_kind_that_names_no_checks_writes_no_failed_checks_key() -> None: + records = build_failed_runs([object()], passed=lambda r: False, detail=lambda r: {}) + assert records[0]["exit_reason"] is None + assert "failed_checks" not in records[0] + + +@dataclass +class _StrRun: + exit_reason: str = "chat_error" + + +def test_a_string_exit_reason_gets_the_same_outcome_as_its_enum() -> None: + # exit_reason and outcome must agree whether a kind records the member or its value. + for exit_reason in (LoopExit.SIMULATED_USER_FAILED, "simulated_user_failed"): + record = build_failed_runs([_StrRun(exit_reason)], passed=lambda r: False, detail=lambda r: {})[0] + assert record["exit_reason"] == "simulated_user_failed" + assert record["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + + +@pytest.mark.parametrize( + "exit_reason", [LoopExit.CHAT_ERROR, "chat_error", "budget_exhausted"], ids=["member", "value", "agent_exit"] +) +def test_a_chat_error_or_agent_exit_is_recorded_and_rendered_as_a_failure(exit_reason: LoopExit | str) -> None: + records = build_failed_runs( + [_StrRun(exit_reason)], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: [] + ) + + value = LoopExit(exit_reason).value + assert records[0]["exit_reason"] == value + assert records[0]["outcome"] == {"status": "fail", "reason": value} + assert failing_runs_note(records, 3) == f"run 1/3 FAIL (loop exit: {value}): no failing check recorded" + + +def test_a_simulated_user_failure_is_recorded_and_rendered_as_an_error() -> None: + records = build_failed_runs( + [_StrRun("simulated_user_failed")], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: [] + ) + + assert records[0]["outcome"]["status"] == "error" + assert failing_runs_note(records, 3) == "run 1/3 ERROR (simulated_user_failed)" + + +@dataclass +class _JudgedRun: + judge_error: str | None + exit_reason: str = "success" + + +def test_a_run_the_judge_could_not_grade_is_recorded_as_an_error() -> None: + (record,) = build_failed_runs([_JudgedRun("empty body twice")], passed=lambda r: False, detail=lambda r: {}) + assert record["outcome"] == {"status": "error", "reason": "judge_failed"} + + +def test_a_labelled_kind_records_the_outcome_its_labels_give() -> None: + """The labels decide both which runs are recorded and their outcome: a run whose judge broke + after it failed a fixed check is labelled FAIL, and its record says so, judge_error or not.""" + runs = [_JudgedRun(None), _JudgedRun("empty body twice"), _JudgedRun(None, exit_reason="simulated_user_failed")] + labels = RunLabels((Outcome.passed(), Outcome.failed("success"), Outcome.error("simulated_user_failed"))) + + records = build_failed_runs(runs, labels=labels, detail=lambda r: {}) + + assert [r["run_index"] for r in records] == [2, 3] + assert records[0]["error"] == "empty body twice" + assert records[0]["outcome"] == {"status": "fail", "reason": "success"} + assert records[1]["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + + +@pytest.mark.parametrize( + "kwargs", + [{}, {"passed": lambda r: False, "labels": RunLabels((Outcome.failed(),))}], + ids=["neither", "both"], +) +def test_failed_runs_take_exactly_one_of_labels_and_passed(kwargs: dict[str, Any]) -> None: + with pytest.raises(ValueError, match="exactly one"): + build_failed_runs([_JudgedRun(None)], detail=lambda r: {}, **kwargs) + + +def test_failed_runs_refuse_labels_for_a_different_number_of_runs() -> None: + with pytest.raises(ValueError, match="1 labels for 2 runs"): + build_failed_runs([_JudgedRun(None)] * 2, labels=RunLabels((Outcome.failed(),)), detail=lambda r: {}) + + +def test_a_failed_check_record_keeps_short_values_and_caps_long_ones() -> None: + long_actual = {"filters": ["x" * 400]} + records = build_failed_runs( + [_StrRun("budget_exhausted")], + passed=lambda r: False, + detail=lambda r: {"actual": long_actual}, + failed_checks=lambda r: [ + {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]}, + {"name": "filters_correct", "part": "attribute", "expected": [], "actual": long_actual}, + ], + ) + + short, long_check = records[0]["failed_checks"] + assert short == {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]} + assert long_check["expected"] == [] + assert isinstance(long_check["actual"], str) + assert len(long_check["actual"]) == 160 + assert long_check["actual"].endswith("...") + # The run's detail keeps the whole value. + assert records[0]["detail"]["actual"] == long_actual + + +def test_a_fail_run_names_its_loop_exit_as_its_reason() -> None: + assert classify_run(LoopExit.SUCCESS, passed=False).reason == "success" + + +def test_a_metric_layout_failure_names_the_layout_problem_not_two_equal_sets() -> None: + ev = MagicMock( + scored_without_chart=False, + visualization_created=True, + cross_ref_valid=True, + metrics_correct=False, + expected_metric_uris={"metric/revenue"}, + actual_metric_uris={"metric/revenue"}, + metric_layout_problems=["metric/revenue is on the secondary axis"], + dimensions_correct=True, + filters_correct=True, + sorts_correct=True, + viz_type_hard=True, + requires_execution=False, + buckets_correct=None, + answer_correct=None, + format_correct=None, + part_present=None, + ) + run = MagicMock(eval_result=ev) + + assert agentic_visualization._failed_checks(run) == [ + {"name": "metrics_correct", "part": "layout", "actual": ["metric/revenue is on the secondary axis"]} + ] + + +# ── visualization: a partial gate failure names the run that failed ────────── + + +def _viz(filter_by: dict | None = None) -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": filter_by or {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn(filter_by: dict | None = None) -> ChatResult: + return ChatResult.model_validate( + {"createdVisualizations": {"objects": [_viz(filter_by)], "reasoning": ""}, "toolCallEvents": []} + ) + + +_LAST_YEAR = {"date": {"type": "date_filter", "using": "dataset/date", "granularity": "MONTH", "from": -11, "to": 0}} + + +def test_a_partial_visualization_failure_lists_the_failing_run_and_keeps_the_best_run_summary() -> None: + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [_viz_turn(), _viz_turn(_LAST_YEAR), _viz_turn()] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=3, + gate="power", + ) + + message = str(exc_info.value) + lines = message.splitlines() + gate_line = next(line for line in lines if line.startswith("Gate pass^3 failed")) + assert gate_line.startswith("Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure. Failing runs: ") + assert "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual " in gate_line + # Only the failing checks of the failing run, and no passing run. + assert "metrics_correct" not in gate_line + assert "run 1/3" not in gate_line + assert "run 3/3" not in gate_line + # The best-run summary that followed the gate note is still there, unchanged. + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + assert "Strict Check Summary:" in message + assert " Filters Correct : True\n" in message + + (record,) = exc_info.value.failed_runs + assert record["run_index"] == 2 + assert record["exit_reason"] == "success" + assert record["detail"]["exit_reason"] == "success" + assert [c["name"] for c in record["failed_checks"]] == ["filters_correct"] + assert record["failed_checks"][0]["part"] == "date" + assert record["failed_checks"][0]["expected"] == [] + + +# ── metric skill: same head, expected/actual MAQL of the failing run ───────── + + +def _metric_turn(maql: str) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": f'{{"data": {{"maql": "{maql}"}}}}', + } + ], + } + ) + + +def test_a_partial_metric_failure_lists_the_failing_run_with_its_maql() -> None: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = [_metric_turn("SELECT {metric/foo}"), _metric_turn("SELECT {metric/bar}")] + + with ( + patch.object(metric_skill_mod, "ChatClient", return_value=client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + gate="power", + ) + + message = str(exc_info.value) + assert message.startswith( + "Metric skill assertion failed. Gate pass^2 failed: 1/2 runs passed — unstable, not a clean failure. " + "Failing runs: run 2/2 FAIL (loop exit: success): " + 'maql_correct[expected ["SELECT {metric/foo}"], actual SELECT {metric/bar}].' + ) + # The best run's summary follows the failing runs. + assert "metric_created=True, maql_correct=True. Expected MAQL (candidates): 'SELECT {metric/foo}'." in message + (record,) = exc_info.value.failed_runs + assert record["exit_reason"] == "success" + assert record["failed_checks"] == [ + {"name": "maql_correct", "expected": ["SELECT {metric/foo}"], "actual": "SELECT {metric/bar}"} + ] + + +def test_a_run_that_created_no_metric_names_only_the_missing_metric() -> None: + """Without a metric, maql_correct reads False only because there is no MAQL to compare.""" + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = ChatResult.model_validate({"textResponse": "I cannot do that."}) + + with ( + patch.object(metric_skill_mod, "ChatClient", return_value=client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=1, + gate="power", + ) + + (record,) = exc_info.value.failed_runs + assert record["failed_checks"] == [{"name": "metric_created", "expected": True, "actual": False}] + assert "maql_correct[" not in str(exc_info.value) + + +@pytest.mark.parametrize("module_name", ["visualization", "alert_skill", "metric_skill", "dashboard_skill"]) +def test_each_checked_kind_heads_its_message_through_the_shared_renderer(module_name: str) -> None: + source = inspect.getsource(importlib.import_module(f"gooddata_eval.core.agentic.{module_name}")) + assert "failure_message_head(gate, run_labels, runs_passed, failed_runs)" in source + assert "failed_checks=_failed_checks" in source + + +# ── the kinds that name their failing checks by name only ───────────────────── + + +def test_the_completion_kinds_name_exactly_the_checks_strict_pass_failed() -> None: + kda = KdaEvaluation(triggered=True, executed=False, success=False, turn_completed=True) + assert kda_failed_checks(SimpleNamespace(evaluation=kda)) == [{"name": "executed"}, {"name": "success"}] + + anomaly = AnomalyEvaluation( + triggered=True, + executed=True, + success=True, + turn_completed=True, + metric_correct=False, + granularity_correct=True, + ) + assert anomaly_failed_checks(SimpleNamespace(evaluation=anomaly)) == [{"name": "metric_correct"}] + assert not anomaly.strict_pass + + what_if = WhatIfEvaluation( + triggered=True, + executed=True, + success=True, + turn_completed=True, + metric_correct=True, + maql_correct=True, + scenario_count_correct=True, + baseline_correct=True, + ) + assert what_if.strict_pass + assert what_if_failed_checks(SimpleNamespace(evaluation=what_if)) == [] + + +def _alert_evaluation(created: bool, **checks: bool) -> AlertEvaluation: + fields = dict.fromkeys( + ( + "operator_correct", + "threshold_correct", + "trigger_correct", + "filters_correct", + "metric_correct", + "recipients_correct", + ), + created, + ) + return AlertEvaluation(alert_created=created, **{**fields, **checks}) + + +def test_an_alert_run_that_created_no_alert_names_only_the_missing_alert() -> None: + """Every other alert check reads False when nothing was created; naming them all would + report argument errors for an alert that does not exist.""" + run = SimpleNamespace(eval=_alert_evaluation(False, title_correct=False)) + assert alert_failed_checks(run) == [{"name": "alert_created"}] + + +def test_an_alert_run_that_created_an_alert_names_each_check_it_failed() -> None: + run = SimpleNamespace(eval=_alert_evaluation(True, threshold_correct=False, title_correct=False)) + assert alert_failed_checks(run) == [{"name": "threshold_correct"}, {"name": "title_correct"}] + + +@pytest.mark.parametrize( + ("failed_checks", "name"), + [(general_question_failed_checks, "general_question_pass"), (guardrail_failed_checks, "guardrail_pass")], +) +def test_a_judged_kind_names_the_judges_reasoning_and_nothing_for_an_ungraded_run( + failed_checks: Callable[[Any], list[dict]], name: str +) -> None: + failed = SimpleNamespace(judge_error=None, passed=False, reasoning="answered a different question") + assert failed_checks(failed) == [{"name": name, "part": "judge", "actual": "answered a different question"}] + ungraded = SimpleNamespace(judge_error="empty body", passed=False, reasoning="") + assert failed_checks(ungraded) == [] diff --git a/packages/gooddata-eval/tests/test_agentic_gate.py b/packages/gooddata-eval/tests/test_agentic_gate.py index e79af9093..bad390e57 100644 --- a/packages/gooddata-eval/tests/test_agentic_gate.py +++ b/packages/gooddata-eval/tests/test_agentic_gate.py @@ -99,6 +99,22 @@ def test_failure_note_blames_the_judge_not_the_agent_for_an_ungraded_run(): assert "unstable" not in note +def test_failure_note_names_harness_errors_without_calling_them_instability(): + note = gate_failure_note("power", 2, 3, runs_errored=1) + assert note == "Gate pass^3 failed: 2/3 runs passed, 1 harness error(s)." + + +def test_failure_note_keeps_an_agent_failure_unstable_beside_a_harness_error(): + """Pass, fail, error: the errored run is named, but the run the agent failed beside a + passing one is still instability.""" + note = gate_failure_note("power", 1, 3, runs_errored=1) + assert note == "Gate pass^3 failed: 1/3 runs passed, 1 harness error(s) — unstable, not a clean failure." + + +def test_failure_note_does_not_call_errored_runs_alone_unstable_under_pass_at_k(): + assert gate_failure_note("any", 0, 3, runs_errored=1) == "Gate pass@3 failed: 0/3 runs passed, 1 harness error(s)." + + def test_gate_scores_use_K_stable_names(): """`pass_at_2` becomes `pass_at_3` the moment K changes, splitting every view on it.""" ctx = MagicMock() diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 6a1690ebb..8f13a4093 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -16,8 +16,8 @@ evaluate_agentic_kda_skill, run_agentic_kda_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.models import ChatResult, LoopExit _EXPECTED = {"Measure": {"type": "metric", "id": "revenue"}} @@ -1296,3 +1296,29 @@ def _capture(_submit, _identity, **kwargs): assert scores["turns"] == 1 assert scores["steps"] == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("gen-ai returned 503"), LoopExit.CHAT_ERROR), + ], +) +def test_run_agentic_kda_skill_names_why_the_chat_call_failed(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + ) + + assert summary.best.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 5efcbb0bb..b8ebb0c63 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -22,7 +22,7 @@ generate_simulated_response, run_agentic_metric_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent from gooddata_eval.core.timing import TIMERS_ENV_VAR @@ -1098,3 +1098,81 @@ def call(mid: str, created_new: bool) -> ToolCallEvent: ) assert _extract_created_metric_ids([call("new", True), call("old", False)]) == ["new"] + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=3, + ) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + + +def test_a_chat_error_run_is_a_failure_scored_as_before(): + mock_client = _client() + mock_client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + } + ), + ] + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + _patched(mock_client), + patch("gooddata_eval.core.agentic.metric_skill.submit_trace_scoring", _capture), + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + run_ids = [MagicMock(name="run-1"), MagicMock(name="run-2")] + ctx.observe.return_value.__enter__.side_effect = run_ids + captured["write_scores"](ctx) + by_run = {id(t): [c.kwargs["name"] for c in ctx.score.call_args_list if c.args[0] is t] for t in run_ids} + + first, second = by_run[id(run_ids[0])], by_run[id(run_ids[1])] + # A chat API error is on the product path: a FAIL, with no harness marker. + assert "eval_error" not in first + assert "gate_passed" in first + assert "eval_error" not in second + assert "gate_passed" in second diff --git a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py index 83f31a277..44a4d4fba 100644 --- a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py +++ b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py @@ -345,3 +345,15 @@ def test_the_run_context_forwards_its_window_and_item_input_to_observe(): "item_input": "Show revenue by month", "output": {"passed": True}, } + + +def test_the_run_context_forwards_a_score_comment_only_when_given(): + lf = MagicMock() + ctx = RunTraceContext({}, lf, "client", "item-1", "base", True, {}, _WINDOW, "q") + + ctx.score("tid", name="gate_passed", value=True, data_type="BOOLEAN") + ctx.score("tid", name="eval_error", value=1, data_type="NUMERIC", comment="backend: the chat call failed") + + plain, commented = lf.score_safe.call_args_list + assert plain.kwargs == {"name": "gate_passed", "value": True, "data_type": "BOOLEAN"} + assert commented.kwargs["comment"] == "backend: the chat call failed" diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index a3653f853..26abd44e2 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -22,7 +22,8 @@ from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.langfuse.item_scope import ItemTraceScope, current_scope -from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, CreatedVisualization, DatasetItem +from gooddata_eval.core.reporting.json_report import build_json_report from gooddata_eval.core.timing import PhaseTimings, note_trace_latency @@ -1022,6 +1023,37 @@ def test_a_kind_that_records_no_failed_runs_keeps_the_empty_default(): assert report.items[0].failed_runs == [] +_HARNESS_ERROR = {"status": "error", "reason": "simulated_user_failed"} +_SHADOW = {"verdict": "error", "runs_scored": 0, "runs_total": 3} + + +def _labelled_failure(exc: AgenticAssertionError | JudgeResponseError) -> Exception: + exc.harness_error = _HARNESS_ERROR + exc.shadow = _SHADOW + return exc + + +@pytest.mark.parametrize( + "result", + [ + pytest.param(AgenticEvalOutcome(reasoning_steps=[], detail={}, shadow=_SHADOW), id="passed"), + pytest.param(_labelled_failure(AlertSkillAssertionError("failed")), id="failed"), + pytest.param(_labelled_failure(JudgeResponseError("no verdict")), id="ungraded"), + ], +) +def test_the_item_labels_reach_the_report(result: Any): + """A label nobody reads labels nothing: the outcome and the shadow verdict travel from + whichever object the kind returned or raised onto the item report and into the JSON.""" + kwargs = {"return_value": result} if isinstance(result, AgenticEvalOutcome) else {"side_effect": result} + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", **kwargs): + report = run_agentic_items([_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") + item = report.items[0] + assert item.shadow == _SHADOW + assert item.harness_error == (None if isinstance(result, AgenticEvalOutcome) else _HARNESS_ERROR) + record = build_json_report(report)["items"][item.id] + assert (record["harness_error"], record["shadow"]) == (item.harness_error, item.shadow) + + def test_an_all_ungraded_item_is_errored_but_still_carries_its_failed_runs(): """JudgeResponseError is a RuntimeError, so it used to land in the generic branch: errored, runs=0, no records. It is still an error -- no run has a verdict -- but the diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index 431c3f329..c50d2fddb 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -18,7 +18,7 @@ evaluate_agentic_visualization, run_agentic_visualization, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.evaluators.visualization import ExpectationOptions from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit @@ -822,3 +822,35 @@ def test_a_failed_request_with_a_cut_off_answer_fails_an_item_without_a_chart() assert result.exit_reason is LoopExit.CHAT_ERROR assert result.eval_result.strict_pass is False assert result.eval_result.answer_problems == ["the chat request failed before the answer ended"] + + +_AGENT_CHAT_FAILURES = [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), +] + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_the_opening_request(exc, loop_exit): + client = MagicMock() + client.send_message.side_effect = exc + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit + assert result.eval_result.visualization_created is False + # Like a chat error, the loop does not run after the opening failure. + assert client.send_message.call_count == 1 + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkeypatch, exc, loop_exit): + monkeypatch.setattr( + "gooddata_eval.core.agentic.visualization.generate_simulated_response", lambda *a, **k: "the revenue one" + ) + client = MagicMock() + client.send_message.side_effect = [_chat_clarification(), exc] + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_what_if.py b/packages/gooddata-eval/tests/test_agentic_what_if.py index 38b04ebf3..4ee0e4ab0 100644 --- a/packages/gooddata-eval/tests/test_agentic_what_if.py +++ b/packages/gooddata-eval/tests/test_agentic_what_if.py @@ -13,6 +13,7 @@ evaluate_agentic_what_if, run_agentic_what_if, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.what_if" @@ -385,7 +386,7 @@ def trace(self, _conversation_id): def observe(self, _trace, _run_idx, *, conversation_id=None): yield "trace-id" - def score(self, _tid, *, name, value, data_type): + def score(self, _tid, *, name, value, data_type, comment=None): self.scores[name] = value def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): @@ -499,3 +500,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_eval_error_scores.py b/packages/gooddata-eval/tests/test_eval_error_scores.py new file mode 100644 index 000000000..38ceab535 --- /dev/null +++ b/packages/gooddata-eval/tests/test_eval_error_scores.py @@ -0,0 +1,162 @@ +# (C) 2026 GoodData Corporation +"""Every agentic loop kind writes its Langfuse scores the same way. + +A run labelled ERROR gets an extra ``eval_error`` score beside its usual ones; every other run, +a chat API error included, gets its usual scores and no ``eval_error``. The visualization kind +has its own tests in test_outcome.py. +""" + +from collections import defaultdict +from collections.abc import Callable, Iterator +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import alert_skill, anomaly_detection, dashboard_skill, kda_skill, what_if +from gooddata_eval.core.agentic._outcome import RunLabels +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.outcome import Outcome + + +@dataclass(frozen=True) +class _Kind: + module: Any + evaluate: Callable[..., Any] + expected_output: dict + simulated_user: str | None + + +_KINDS = { + "alert_skill": _Kind( + alert_skill, + alert_skill.evaluate_agentic_alert_skill, + {"operator": "GREATER_THAN", "threshold": 100}, + "generate_simulated_alert_response", + ), + "kda_skill": _Kind( + kda_skill, + kda_skill.evaluate_agentic_kda_skill, + {"Measure": {"type": "metric", "id": "revenue"}}, + "generate_simulated_kda_response", + ), + "dashboard_skill": _Kind( + dashboard_skill, + dashboard_skill.evaluate_agentic_dashboard_skill, + { + "type": "dashboard", + "saved_dashboard_id": None, + "visualizations": [{"id": None, "title": "Revenue"}], + "date_range": None, + "min_new_visualizations": 1, + }, + None, + ), + "anomaly_detection": _Kind( + anomaly_detection, + anomaly_detection.evaluate_agentic_anomaly_detection, + {"metric": "metric/spend", "granularity": "MONTH"}, + "generate_simulated_anomaly_response", + ), + "what_if": _Kind( + what_if, + what_if.evaluate_agentic_what_if, + {"metric_id": "revenue"}, + "generate_simulated_what_if_response", + ), +} + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + def run_name(self, run_idx: int) -> str: + return f"run{run_idx}" + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(kind: _Kind, labels: RunLabels | None = None) -> _ScoreCtx: + """Run 1's chat call fails after its retries; run 2 answers without doing the task. + + ``labels`` replaces the labels the kind computes, to give run 1 a harness ERROR without + driving each kind's own simulated-user loop.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate({"textResponse": "I cannot help with that."}), + ] + captured: dict[str, Any] = {} + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + captured["write_scores"] = kwargs["write_scores"] + + with ExitStack() as stack: + stack.enter_context(patch.object(kind.module, "ChatClient", return_value=client)) + stack.enter_context(patch.object(kind.module, "submit_trace_scoring", _capture)) + if labels is not None: + stack.enter_context(patch.object(kind.module, "label_runs", return_value=labels)) + if kind.simulated_user is not None: + stack.enter_context(patch.object(kind.module, kind.simulated_user, return_value="please go on")) + try: + kind.evaluate( + "http://host/api/v1/actions/workspaces/ws1/ai", + "tok", + "ws1", + "Do the thing", + kind.expected_output, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + except AssertionError: + pass # the scored run fails; its scores are written before the raise + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +_USUAL_SCORES = {"pass_at_k", "pass_power_k", "gate_passed"} + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_an_errored_run_gets_eval_error_beside_its_usual_scores(name: str) -> None: + labels = RunLabels((Outcome.error("simulated_user_failed"), Outcome.failed("agent_silent"))) + ctx = _scores(_KINDS[name], labels) + + assert ("eval_error", 1, "simulated_user_failed") in ctx.scores[0] + assert set(ctx.names(0)) >= _USUAL_SCORES + assert "eval_error" not in ctx.names(1) + assert set(ctx.names(1)) >= _USUAL_SCORES + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_a_chat_error_run_is_scored_as_before_with_no_marker(name: str) -> None: + ctx = _scores(_KINDS[name]) + + for run_idx in (0, 1): + names = ctx.names(run_idx) + assert "eval_error" not in names + assert set(names) >= _USUAL_SCORES diff --git a/packages/gooddata-eval/tests/test_outcome.py b/packages/gooddata-eval/tests/test_outcome.py new file mode 100644 index 000000000..0fb00a545 --- /dev/null +++ b/packages/gooddata-eval/tests/test_outcome.py @@ -0,0 +1,539 @@ +# (C) 2026 GoodData Corporation +"""PASS / FAIL / ERROR labels per run: they explain, they never rescore. + +A run is ERROR only when the harness failed -- its simulated user, or its judge when the judge's +verdict was the one still open. Every other non-pass, a chat API error and the agent's own +timeout included, is FAIL. The labels change no gate or count: an ERROR run still counts as a +run that did not pass. The message names the ERROR runs and counts them in its gate note. +""" + +from collections import defaultdict +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.agentic._outcome import ( + RunLabels, + chat_failure_exit, + failure_message_head, + label_runs, + mark_unscored_run, +) +from gooddata_eval.core.agentic.document_skill import _judge_error_of +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + CreatedVisualization, + LoopExit, +) +from gooddata_eval.core.outcome import ( + JUDGE_FAILED, + Outcome, + OutcomeStatus, + classify_run, + item_outcome, + shadow_verdict, +) + +PASS = Outcome.passed() +FAIL = Outcome.failed("agent_silent") +SIM = Outcome.error("simulated_user_failed") +JUDGE = Outcome.error(JUDGE_FAILED) + + +# ── the rule: an allowlist of harness causes, everything else is FAIL ───────── + +# Every LoopExit, with the label of a run that ended on it without passing. Exhaustive on +# purpose: a new member must be classified here before the suite passes. +_EXPECTED = { + LoopExit.SUCCESS: OutcomeStatus.FAIL, + LoopExit.AGENT_SILENT: OutcomeStatus.FAIL, + LoopExit.BUDGET_EXHAUSTED: OutcomeStatus.FAIL, + LoopExit.STOPPED_EARLY: OutcomeStatus.FAIL, + LoopExit.SIMULATED_USER_FAILED: OutcomeStatus.ERROR, + LoopExit.CHAT_ERROR: OutcomeStatus.FAIL, + LoopExit.AGENT_TIMEOUT: OutcomeStatus.FAIL, + LoopExit.TURN_INCOMPLETE: OutcomeStatus.FAIL, + LoopExit.NOT_RUN: OutcomeStatus.FAIL, +} + + +def test_every_loop_exit_is_classified() -> None: + assert set(_EXPECTED) == set(LoopExit) + + +@pytest.mark.parametrize("loop_exit", list(LoopExit)) +def test_each_loop_exit_has_its_label_and_names_itself_as_the_reason(loop_exit: LoopExit) -> None: + outcome = classify_run(loop_exit, passed=False) + assert outcome.status is _EXPECTED[loop_exit] + assert outcome.reason == loop_exit.value + assert classify_run(loop_exit.value, passed=False) == outcome + + +@pytest.mark.parametrize("loop_exit", list(LoopExit)) +def test_a_run_that_passed_is_pass_whatever_its_exit(loop_exit: LoopExit) -> None: + assert classify_run(loop_exit, passed=True) == PASS + + +def test_a_judge_failure_is_an_error_whatever_the_exit() -> None: + assert classify_run(LoopExit.SUCCESS, passed=False, judge_error="empty body twice") == JUDGE + assert classify_run(None, passed=False, judge_error="unparseable JSON") == JUDGE + + +def test_an_unknown_cause_is_a_failure_never_an_error() -> None: + assert classify_run("some_new_exit", passed=False) == Outcome.failed("some_new_exit") + assert classify_run(None, passed=False) == Outcome.failed(None) + + +def test_an_outcome_record_is_json_ready() -> None: + assert SIM.as_record() == {"status": "error", "reason": "simulated_user_failed"} + assert Outcome.failed("chat_error").as_record() == {"status": "fail", "reason": "chat_error"} + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("503 after retries"), LoopExit.CHAT_ERROR), + (ChatError("SSE stream error"), LoopExit.CHAT_ERROR), + (httpx.ReadError("connection reset"), LoopExit.CHAT_ERROR), + ], +) +def test_a_chat_failure_maps_to_its_own_exit_and_is_a_failure(exc: BaseException, loop_exit: LoopExit) -> None: + assert chat_failure_exit(exc) is loop_exit + assert classify_run(chat_failure_exit(exc), passed=False).status is OutcomeStatus.FAIL + + +# ── the item: ERROR only when no run was scored ────────────────────────────── + + +def test_an_item_is_an_error_only_when_every_run_errored() -> None: + assert item_outcome([SIM, SIM, SIM]) == SIM + assert item_outcome([SIM, PASS, PASS]) is None + assert item_outcome([SIM, FAIL]) is None + assert item_outcome([]) is None + + +def test_an_errored_item_names_its_most_frequent_reason() -> None: + assert item_outcome([SIM, JUDGE, JUDGE]) == JUDGE + + +# ── the shadow verdict: the phase-2 candidate rule, recorded and never used ── + + +@pytest.mark.parametrize( + ("outcomes", "gate", "verdict", "scored"), + [ + ([SIM, PASS, PASS], "power", "pass", 2), + ([SIM, SIM, PASS], "power", "error", 1), # fewer than ceil(3/2) scored runs + ([SIM, SIM, SIM], "power", "error", 0), + ([Outcome.failed("chat_error"), PASS, PASS], "power", "fail", 3), # a chat error is scored + ([SIM, PASS], "power", "pass", 1), # K=2 needs one scored run + ([SIM], "power", "error", 0), + ([SIM, FAIL, PASS], "any", "pass", 2), + ([PASS, PASS, PASS], "power", "pass", 3), + ], +) +def test_the_shadow_verdict(outcomes: list[Outcome], gate: str, verdict: str, scored: int) -> None: + assert shadow_verdict(outcomes, gate) == {"verdict": verdict, "runs_scored": scored, "runs_total": len(outcomes)} + + +class _Run: + def __init__(self, ok: bool, exit_reason: LoopExit | None = None, judge_error: str | None = None) -> None: + self.ok = ok + self.exit_reason = exit_reason + self.judge_error = judge_error + + +def test_label_runs_reads_each_runs_exit_and_judge() -> None: + runs = [ + _Run(False, LoopExit.SIMULATED_USER_FAILED), + _Run(False, LoopExit.SUCCESS, judge_error="no verdict"), + _Run(True, LoopExit.SUCCESS), + _Run(False, LoopExit.CHAT_ERROR), + ] + labels = label_runs(runs, passed=lambda r: r.ok) + assert labels.outcomes == (SIM, JUDGE, PASS, Outcome.failed("chat_error")) + assert labels.error_label(0) == SIM + assert labels.error_label(2) is None + assert labels.error_label(3) is None + assert labels.error_label(9) is None + assert labels.item_outcome is None + assert labels.shadow("power") == {"verdict": "fail", "runs_scored": 2, "runs_total": 4} + + +def test_a_kind_can_name_its_own_judge_error() -> None: + run = _Run(False, LoopExit.SUCCESS, judge_error="takeaway request failed") + assert label_runs([run], passed=lambda r: r.ok).outcomes == (JUDGE,) + labels = label_runs([run], passed=lambda r: r.ok, judge_error=lambda r: None) + assert labels.outcomes == (Outcome.failed("success"),) + + +class _Evaluation: + def __init__(self, judge_error: str | None, fill_error: str | None, checks_held: bool) -> None: + self.judge_error = judge_error + self.fill_error = fill_error + self.ungraded = (judge_error is not None or fill_error is not None) and checks_held + + +class _DocRun: + def __init__(self, judge_error: str | None, fill_error: str | None, checks_held: bool = True) -> None: + self.evaluation = _Evaluation(judge_error, fill_error, checks_held) + self.exit_reason = None + + +def test_a_document_run_whose_takeaway_request_failed_is_a_failure_not_a_judge_error() -> None: + request_failed = _DocRun(judge_error=None, fill_error="503 from the takeaway endpoint") + judge_failed = _DocRun(judge_error="empty body twice", fill_error=None) + labels = label_runs([request_failed, judge_failed], passed=lambda r: False, judge_error=_judge_error_of) + assert labels.outcomes == (Outcome.failed(None), JUDGE) + + +def test_a_document_run_that_failed_a_fixed_check_is_a_failure_even_when_the_judge_errors() -> None: + run = _DocRun(judge_error="empty body twice", fill_error=None, checks_held=False) + labels = label_runs([run], passed=lambda r: False, judge_error=_judge_error_of) + assert labels.outcomes == (Outcome.failed(None),) + + +def test_an_unscored_run_gets_the_marker_only_when_the_harness_failed() -> None: + ctx = MagicMock() + mark_unscored_run(ctx, 0, "conv-1", JUDGE) + ctx.export_unscored.assert_not_called() + ctx.score.assert_called_once() + assert ctx.score.call_args.kwargs == { + "name": "eval_error", + "value": 1, + "data_type": "NUMERIC", + "comment": "judge_failed", + } + + ctx = MagicMock() + mark_unscored_run(ctx, 1, "conv-2", None) + ctx.export_unscored.assert_called_once_with(1, "conv-2") + ctx.score.assert_not_called() + + +# ── end to end through the visualization kind: labels, never a new verdict ─── + + +def _viz() -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn() -> ChatResult: + return ChatResult.model_validate({"createdVisualizations": {"objects": [_viz()], "reasoning": ""}}) + + +def _question_turn() -> ChatResult: + return ChatResult.model_validate({"textResponse": "Which metric?"}) + + +def _evaluate(turns: list[Any], captured: dict[str, Any] | None = None, gate: str = "power") -> object: + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, len(turns) + 1)] + client.send_message.side_effect = turns + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + if captured is not None: + captured["write_scores"] = kwargs["write_scores"] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + patch.object(agentic_visualization, "generate_simulated_response", side_effect=RuntimeError("openai 401")), + patch.object(agentic_visualization, "submit_trace_scoring", _capture), + ): + return evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=len(turns), + gate=gate, + langfuse=MagicMock() if captured is not None else None, + dataset_item_id="item-1" if captured is not None else None, + ) + + +def _silent_turn() -> ChatResult: + return ChatResult.model_validate({}) + + +def test_a_simulated_user_failure_keeps_the_verdict_and_is_labelled() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _viz_turn(), _viz_turn()]) + + exc = exc_info.value + message = str(exc) + lines = message.splitlines() + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + # The gate counts the errored run as a run that did not pass: pass^3, 2/3. The note does + # not call that instability: the agent was not what failed it. + assert lines[2].startswith(gate_failure_note("power", 2, 3, runs_errored=1)) + assert "unstable" not in lines[2] + assert "run 1/3 ERROR (simulated_user_failed): visualization_created[expected true, actual false]" in lines[2] + assert (exc.runs_passed, exc.runs_effective) == (2, 3) + (record,) = exc.failed_runs + assert record["exit_reason"] == "simulated_user_failed" + assert record["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + # Some runs were scored, so the item is not a harness error. + assert not hasattr(exc, "harness_error") + assert exc.shadow == {"verdict": "pass", "runs_scored": 2, "runs_total": 3} + + +def test_a_passing_item_records_its_shadow_verdict_too() -> None: + # pass@3 passes on the one scored run; the candidate rule, wanting ceil(3/2) scored runs, + # would not. Only a shadow recorded on a passing item shows that. + outcome = _evaluate([_question_turn(), _question_turn(), _viz_turn()], gate="any") + assert isinstance(outcome, AgenticEvalOutcome) + assert outcome.shadow == {"verdict": "error", "runs_scored": 1, "runs_total": 3} + assert [r["outcome"]["status"] for r in outcome.failed_runs] == ["error", "error"] + + +def test_an_item_whose_every_run_errored_is_labelled_and_still_fails() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _question_turn(), _question_turn()]) + + exc = exc_info.value + assert isinstance(exc, AgenticAssertionError) + assert exc.harness_error == {"status": "error", "reason": "simulated_user_failed"} + assert exc.shadow == {"verdict": "error", "runs_scored": 0, "runs_total": 3} + gate_line = str(exc).splitlines()[2] + assert gate_line.startswith( + f"Harness error on every run (simulated_user_failed). {gate_failure_note('power', 0, 3, runs_errored=3)}" + ) + assert exc.runs_passed == 0 + assert len(exc.failed_runs) == 3 + + +def test_the_head_says_no_run_was_scored_only_when_every_judge_failed() -> None: + labels = RunLabels((Outcome.error(JUDGE_FAILED), Outcome.error(JUDGE_FAILED))) + head = failure_message_head("power", labels, runs_passed=0, failed_runs=[], runs_ungraded=2) + assert head.startswith("Harness error, no run was scored (judge_failed). ") + + +def test_the_head_does_not_say_unscored_when_a_simulated_user_failed() -> None: + """A run whose simulated user failed was still checked; its failed checks follow the head.""" + labels = RunLabels((Outcome.error("simulated_user_failed"), Outcome.error(JUDGE_FAILED))) + head = failure_message_head("power", labels, runs_passed=0, failed_runs=[], runs_ungraded=1) + assert head.startswith("Harness error on every run (judge_failed). ") + assert "no run was scored" not in head + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + (ChatError("SSE stream error"), "chat_error"), + ], +) +def test_a_chat_failure_is_a_plain_failure(exc: BaseException, exit_value: str) -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([exc, _viz_turn(), _viz_turn()]) + + raised = exc_info.value + assert (raised.runs_passed, raised.runs_effective) == (2, 3) + (record,) = raised.failed_runs + assert record["exit_reason"] == exit_value + assert record["outcome"] == {"status": "fail", "reason": exit_value} + assert f"run 1/3 FAIL (loop exit: {exit_value})" in str(raised) + assert "ERROR" not in str(raised) + assert not hasattr(raised, "harness_error") + assert raised.shadow["verdict"] == "fail" + + +def test_only_timeouts_is_a_failure_not_a_harness_error() -> None: + timeout = TurnTimeoutError("agent exceeded the 60s turn budget") + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([timeout, timeout, timeout]) + + assert not hasattr(exc_info.value, "harness_error") + assert str(exc_info.value).splitlines()[2].startswith(gate_failure_note("power", 0, 3)) + + +# ── Langfuse: an ERROR run gets an extra marker, every usual score stays ────── + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(turns: list[Any]) -> _ScoreCtx: + captured: dict[str, Any] = {} + try: + _evaluate(turns, captured) + except AssertionError: + pass + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +_USUAL_SCORES = {"gate_passed", "pass_at_k", "pass_power_k", "quality_score"} + + +def test_an_errored_run_gets_eval_error_beside_its_usual_scores() -> None: + ctx = _scores([_question_turn(), _viz_turn(), _viz_turn()]) + + assert ("eval_error", 1, "simulated_user_failed") in ctx.scores[0] + assert set(ctx.names(0)) >= _USUAL_SCORES + # The verdict scores are the ones the item was judged by: unchanged by the label. + assert dict((n, v) for n, v, _ in ctx.scores[0])["gate_passed"] is False + for run_idx in (1, 2): + assert "eval_error" not in ctx.names(run_idx) + assert set(ctx.names(run_idx)) >= _USUAL_SCORES + + +def test_a_chat_error_run_gets_no_marker() -> None: + ctx = _scores([ChatError("SSE stream error"), _viz_turn(), _viz_turn()]) + + for run_idx in range(3): + assert "eval_error" not in ctx.names(run_idx) + assert set(ctx.names(run_idx)) >= _USUAL_SCORES + + +# ── the message: the gate note and the failing runs, then the best run ─────── + +# The message of [simulated user failed, agent silent, pass] under pass^3, with the "Failing +# runs" part of its gate line left out. The best run's summary follows the gate line. +_MESSAGE_WITHOUT_FAILING_RUNS = ( + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + "Agentic Visualization Assertion Failed! (Critical Mode)\n" + "Gate pass^3 failed: 1/3 runs passed, 1 harness error(s) — unstable, not a clean failure.\n" + "------------------------------------------\n" + "Question:\n" + "Show revenue by quarter\n" + "------------------------------------------\n" + "Expected Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Actual Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Strict Check Summary:\n" + " Visualization Created : True\n" + " Cross-Ref Valid : True\n" + " Metrics Correct : True\n" + " expected : ['metric/revenue']\n" + " actual : ['metric/revenue']\n" + " Dimensions Correct : True\n" + " expected : ['label/date.quarter']\n" + " actual : ['label/date.quarter']\n" + " Filters Correct : True\n" + " date : True\n" + " ranking : True\n" + " attribute : True\n" + " Viz Type Hard : True\n" + " Executed : False\n" + " Stated Value Matches : None (reported only)\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" +) + + +def test_the_agentic_message_is_the_gate_note_the_failing_runs_and_the_best_run() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _silent_turn(), _viz_turn()]) + + message = str(exc_info.value) + gate_line = message.splitlines()[2] + head, failing = gate_line.split(" Failing runs: ", 1) + # A chartless run names only the missing chart, not every chart check that read False. + assert failing.startswith( + "run 1/3 ERROR (simulated_user_failed): visualization_created[expected true, actual false]; " + ) + assert failing.endswith( + "; run 2/3 FAIL (loop exit: agent_silent): visualization_created[expected true, actual false]." + ) + # The gate note names the errored run, and run 2 failed on its own beside a passing run 3, + # so the item is still unstable. + assert head == "Gate pass^3 failed: 1/3 runs passed, 1 harness error(s) — unstable, not a clean failure." + assert message.replace(gate_line, head, 1) == _MESSAGE_WITHOUT_FAILING_RUNS diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 629585358..6f28a767a 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -1091,6 +1091,73 @@ def forever(): assert sse_mod._is_retryable_exc(sse_mod.TurnTimeoutError("x")) is False +def _silent_after_one_line(now: list[float], silent_until: float): + """One event, then a read timeout that ends with the clock at ``silent_until``.""" + yield 'data: {"role":"assistant","content":{"type":"reasoning","text":"thinking"}}' + now[0] = silent_until + raise httpx.ReadTimeout("timed out") + + +def test_a_silent_agent_past_its_budget_is_a_turn_timeout(monkeypatch): + """The read timeout is lowered to the budget, so a silent agent ends in a read timeout, + not at the deadline check -- it is the same timeout all the same.""" + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + lines = sse_mod._until_deadline(_silent_after_one_line(now, 61.0), deadline=60.0, budget=60.0) + with pytest.raises(sse_mod.TurnTimeoutError, match="exceeded the 60s turn budget") as exc_info: + sse_mod.parse_sse_lines(lines) + assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) + + +def test_a_read_timeout_before_the_deadline_stays_a_stream_error(monkeypatch): + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + lines = sse_mod._until_deadline(_silent_after_one_line(now, 10.0), deadline=60.0, budget=60.0) + with pytest.raises(sse_mod.ChatError) as exc_info: + sse_mod.parse_sse_lines(lines) + assert not isinstance(exc_info.value, sse_mod.TurnTimeoutError) + assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) + + +def _headers_never_came(now: list[float], silent_until: float): + def handler(request): + now[0] = silent_until + raise httpx.ReadTimeout("timed out", request=request) + + return handler + + +def test_a_read_timeout_awaiting_the_headers_past_the_budget_is_a_turn_timeout(monkeypatch): + """A server that holds the headers back until the first event times out before the body.""" + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + client = _client_with_handler(_headers_never_came(now, 61.0), turn_timeout_s=60) + with pytest.raises(sse_mod.TurnTimeoutError, match="exceeded the 60s turn budget") as exc_info: + client.send_message("conv", "q") + assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) + + +def test_a_read_timeout_awaiting_the_headers_past_the_budget_records_the_turn(monkeypatch): + """The turn carries no partial result, but it is one of the slowest: the tail needs it.""" + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + client = _client_with_handler(_headers_never_came(now, 61.0), turn_timeout_s=60) + recorder = TraceRecorder() + with pytest.raises(sse_mod.TurnTimeoutError): + client.send_message("conv", "q") + assert recorder.stop() == [pytest.approx(61.0)] + + +def test_a_read_timeout_awaiting_the_headers_before_the_budget_is_left_as_it_is(monkeypatch): + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + client = _client_with_handler(_headers_never_came(now, 10.0), turn_timeout_s=60) + recorder = TraceRecorder() + with pytest.raises(httpx.ReadTimeout): + client.send_message("conv", "q") + assert recorder.stop() == [] + + def test_no_turn_timeout_leaves_the_stream_untouched(): lines = ['data: {"role":"assistant","content":{"type":"text","text":"hi"}}'] assert list(sse_mod._until_deadline(iter(lines), deadline=None)) == lines