diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 183cb387c..4b33b57a2 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -34,6 +34,7 @@ experiment. The newest and most actively developed package in the repo. | `core/reporting/` | console and JSON output rendering | | `core/scoring.py`, `core/runner.py` | scoring and orchestration | | `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends | +| `core/outcome.py` | `Outcome` (PASS / FAIL / ERROR): the one fail-closed rule that labels a failing run ERROR when the harness failed (simulated user, judge), plus the item label and the recorded-only shadow verdict. Labels decide no verdict; the agentic kinds reach it through `core/agentic/_outcome.py`'s `label_runs` | **Depends on**: `gooddata-sdk`, `httpx`, `pydantic`, `orjson`, `rich`. The LLM-judge evaluator is an optional extra (`llm-judge`, pulling `openai>=1.45,<2.0`); every `openai` diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py index 694928b9b..fd056088d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py @@ -25,6 +25,8 @@ from collections.abc import Callable, Sequence from typing import Any +from gooddata_eval.core.agentic._failure_message import cap_failed_checks +from gooddata_eval.core.agentic._outcome import JudgeErrorOf, run_exit_reason, run_outcome from gooddata_eval.core.models import build_tool_calls @@ -33,6 +35,8 @@ def build_failed_runs( *, passed: Callable[[Any], bool], detail: Callable[[Any], dict], + failed_checks: Callable[[Any], list[dict]] | None = None, + judge_error: JudgeErrorOf | None = None, ) -> list[dict]: """One record per run that did not pass, in the order the runs happened. @@ -47,6 +51,14 @@ def build_failed_runs( A run the judge could not grade is recorded like any other non-passing run, with its ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``. + + ``exit_reason`` is the run's ``LoopExit`` value, or null for a kind whose loop records + none, and ``outcome`` its ``core.outcome.Outcome`` record: ERROR when the harness failed + (its simulated user or its judge), FAIL otherwise. The label decides nothing; the run + counts as not passed either way; ``judge_error`` is passed on to ``run_outcome``. ``failed_checks`` is written only when the caller supplies it: a missing + key means the kind does not name its failing checks, while an empty list would claim the + run failed none. Its expected/actual values are capped in size; the run's ``detail`` + keeps them whole. """ records: list[dict] = [] for run_index, run in enumerate(run_results, start=1): @@ -54,19 +66,22 @@ def build_failed_runs( continue tool_calls = list(getattr(run, "tool_call_events", None) or []) reasoning_steps = list(getattr(run, "reasoning_steps", None) or []) - records.append( - { - "run_index": run_index, - "passed": False, - "error": getattr(run, "judge_error", None), - "detail": detail(run), - "conversation_id": getattr(run, "conversation_id", None), - "response_id": getattr(run, "response_id", None), - "reasoning_step_count": len(reasoning_steps), - "reasoning_steps": reasoning_steps, - "tool_call_count": len(tool_calls), - "tool_names": [getattr(e, "function_name", None) for e in tool_calls], - "tool_calls": build_tool_calls(tool_calls), - } - ) + record: dict[str, Any] = { + "run_index": run_index, + "passed": False, + "error": getattr(run, "judge_error", None), + "detail": detail(run), + "conversation_id": getattr(run, "conversation_id", None), + "response_id": getattr(run, "response_id", None), + "reasoning_step_count": len(reasoning_steps), + "reasoning_steps": reasoning_steps, + "tool_call_count": len(tool_calls), + "tool_names": [getattr(e, "function_name", None) for e in tool_calls], + "tool_calls": build_tool_calls(tool_calls), + "exit_reason": run_exit_reason(run), + "outcome": run_outcome(run, passed=False, judge_error=judge_error).as_record(), + } + if failed_checks is not None: + record["failed_checks"] = cap_failed_checks(failed_checks(run)) + records.append(record) return records diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py new file mode 100644 index 000000000..b7c8ae902 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py @@ -0,0 +1,142 @@ +# (C) 2026 GoodData Corporation +"""The head of an agentic kind's failure message: the gate result, then every failing run. + +Each kind's assertion message describes its BEST run, which on a partial gate failure (some +of the K runs passed) is a passing run: every check reads True and the reason the item +failed is nowhere in the message. This module renders the part that is missing -- for each +failing run its index, why its loop stopped and the checks it failed, with expected/actual +where the kind knows them -- from the same ``failed_runs`` records the item reports, so the +message and the triage data cannot disagree about which runs failed or why. + +One renderer for every kind on purpose: the readers of these messages (people and the +triage tooling) look for one shape, whatever the capability under test. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +__all__ = ["FailedCheck", "cap_failed_checks", "failed_check_records", "failing_runs_note", "failure_head"] + +# A value longer than this is cut: the message is one line per run, and the whole value is in +# the run's detail anyway. +_MAX_VALUE_CHARS = 160 + + +class _Unset(Enum): + UNSET = "unset" + + +_UNSET = _Unset.UNSET + + +@dataclass(frozen=True) +class FailedCheck: + """One check a run failed, with the expected/actual pair when the kind knows it. + + ``label`` narrows the check when one name covers several parts, e.g. the ``date`` part of + ``filters_correct``. + """ + + name: str + expected: Any = _UNSET + actual: Any = _UNSET + label: str | None = None + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into a failing run's ``failed_checks``.""" + record: dict[str, Any] = {"name": self.name} + if self.label is not None: + record["label"] = self.label + if self.expected is not _UNSET: + record["expected"] = self.expected + if self.actual is not _UNSET: + record["actual"] = self.actual + return record + + +def failed_check_records(checks: Iterable[FailedCheck]) -> list[dict[str, Any]]: + """``checks`` as the records ``failed_runs[].failed_checks`` holds.""" + return [check.as_record() for check in checks] + + +def _text(value: Any) -> str: + text = value if isinstance(value, str) else json.dumps(value, default=str, ensure_ascii=False, sort_keys=True) + return " ".join(text.split()) + + +def _value(value: Any) -> str: + if value is None or value == [] or value == {} or value == "": + return "none" + text = _text(value) + return text if len(text) <= _MAX_VALUE_CHARS else text[: _MAX_VALUE_CHARS - 3] + "..." + + +def cap_failed_checks(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """``records`` with every expected/actual value cut to what the message shows of it. + + A check's values can be whole filter or bucket structures, repeated for every failing run + of every item; the record keeps the same ``_MAX_VALUE_CHARS`` the message does. A value + that fits stays as it is, JSON and all; a longer one becomes its cut text. The run's + ``detail`` holds the full values. + """ + capped: list[dict[str, Any]] = [] + for record in records: + entry = dict(record) + for key in ("expected", "actual"): + if key in entry and len(_text(entry[key])) > _MAX_VALUE_CHARS: + entry[key] = _value(entry[key]) + capped.append(entry) + return capped + + +def _render_check(record: Mapping[str, Any]) -> str: + name = str(record.get("name", "?")) + parts: list[str] = [] + if "expected" in record: + parts.append(f"expected {_value(record['expected'])}") + if "actual" in record: + parts.append(f"actual {_value(record['actual'])}") + label = record.get("label") + if not parts and label is None: + return name + inner = ", ".join(parts) + if label is not None: + inner = f"{label}: {inner}" if inner else str(label) + return f"{name}[{inner}]" + + +def _render_run(record: Mapping[str, Any], runs_total: int) -> str: + outcome = record.get("outcome") + head = f"run {record.get('run_index', '?')}/{runs_total}" + exit_reason = record.get("exit_reason") + if isinstance(outcome, Mapping) and outcome.get("status") == "error": + # A harness error: the reason (simulated_user_failed, judge_failed) is the diagnosis. + # Its checks are not listed: they failed because the harness did, not the agent. + return f"{head} ERROR ({outcome.get('reason') or 'unknown'})" + head += " FAIL" + (f" (loop exit: {exit_reason})" if exit_reason else "") + checks = record.get("failed_checks") + if checks: + return f"{head}: {', '.join(_render_check(c) for c in checks)}" + if checks is not None: + return f"{head}: no failing check recorded" + return head + + +def failing_runs_note(failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """Every failing run on one line each, joined with ``; ``. Empty when no run failed.""" + return "; ".join(_render_run(record, runs_total) for record in failed_runs) + + +def failure_head(gate_note: str, failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """``gate_note`` followed by the failing runs, ready to start a kind's assertion message. + + Returns ``gate_note`` unchanged when there is no failing run to list, so a kind that + builds no ``failed_runs`` keeps its message exactly as before. + """ + note = failing_runs_note(failed_runs, runs_total) + return f"{gate_note} Failing runs: {note}." if note else gate_note diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py index 576aee838..0a663675a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py @@ -19,18 +19,171 @@ from __future__ import annotations +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum from typing import Any, NoReturn +from gooddata_eval.core.agentic._failure_message import failure_head +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, + LoopExit, ReasoningStepEvent, ToolCallEvent, timeline_detail, ) +from gooddata_eval.core.outcome import EVAL_ERROR_SCORE, Outcome, classify_run, item_outcome, shadow_verdict from gooddata_eval.core.timing import PhaseTimings -__all__ = ["agentic_detail", "agentic_success", "raise_agentic_failure"] +__all__ = [ + "CHAT_FAILURE_EXITS", + "EVAL_ERROR_SCORE", + "RunLabels", + "agentic_detail", + "agentic_success", + "attach_labels", + "chat_failure_exit", + "gate_head", + "label_runs", + "log_eval_error_score", + "mark_unscored_run", + "raise_agentic_failure", + "run_exit_reason", + "run_outcome", +] + + +CHAT_FAILURE_EXITS = frozenset({LoopExit.CHAT_ERROR, LoopExit.AGENT_TIMEOUT, LoopExit.TURN_INCOMPLETE}) +"""Every exit ``chat_failure_exit`` can return: the loop stopped because a chat call raised.""" + + +def chat_failure_exit(exc: BaseException) -> LoopExit: + """The ``LoopExit`` for a run whose chat call raised ``exc``. + + ``TurnTimeoutError`` and ``TurnIncompleteError`` subclass ``ChatError`` but are the + agent's own failures -- too slow, or no final answer -- so they get exits of their own + instead of reading as a chat API fault. Anything else a kind catches there (a gen-ai + error event, a broken stream, a 5xx after its retries) is ``CHAT_ERROR``. All three are + FAIL in ``core/outcome.py``: a customer hits each of them too. + """ + if isinstance(exc, TurnTimeoutError): + return LoopExit.AGENT_TIMEOUT + if isinstance(exc, TurnIncompleteError): + return LoopExit.TURN_INCOMPLETE + return LoopExit.CHAT_ERROR + + +def run_exit_reason(run: Any) -> str | None: + """The run's ``exit_reason`` as a string, or None for a kind whose run records none.""" + value = getattr(run, "exit_reason", None) + if isinstance(value, Enum): + return str(value.value) + return value if isinstance(value, str) else None + + +JudgeErrorOf = Callable[[Any], str | None] +"""How a kind reads a run's judge failure. The default reads ``run.judge_error``; a kind whose +``judge_error`` also reports a product-path failure passes the judge's own error instead.""" + + +def _default_judge_error(run: Any) -> str | None: + value = getattr(run, "judge_error", None) + return value if isinstance(value, str) else None + + +def run_outcome(run: Any, *, passed: bool, judge_error: JudgeErrorOf | None = None) -> Outcome: + """One run's label: ``classify_run`` over the run's exit and its judge failure, if any.""" + error = (judge_error or _default_judge_error)(run) + return classify_run(run_exit_reason(run), passed=passed, judge_error=error if isinstance(error, str) else None) + + +@dataclass(frozen=True) +class RunLabels: + """The label of every run of an item, in run order. + + The labels decide nothing: the kind gates on its own pass_at_k/pass_power_k, and an + ERROR run counts as a run that did not pass. They only say why a run did not pass, and + whether the whole item is a harness error (``item_outcome``). + """ + + outcomes: tuple[Outcome, ...] + + def errored_run(self, run_index: int) -> Outcome | None: + """The ERROR label of run ``run_index`` (0-based), or None when the run is not one.""" + outcome = self.outcomes[run_index] if 0 <= run_index < len(self.outcomes) else None + return outcome if outcome is not None and outcome.is_error else None + + @property + def item_outcome(self) -> Outcome | None: + """ERROR when every run errored, else None (see ``core.outcome.item_outcome``).""" + return item_outcome(self.outcomes) + + @property + def outcome_record(self) -> dict[str, Any] | None: + """``item_outcome`` as the record the exception carries, or None.""" + outcome = self.item_outcome + return outcome.as_record() if outcome is not None else None + + def shadow(self, gate: str | None) -> dict[str, Any]: + """The phase-2 candidate verdict, recorded on the exception and never used.""" + return shadow_verdict(self.outcomes, gate) + + +def label_runs( + run_results: Sequence[Any], *, passed: Callable[[Any], bool], judge_error: JudgeErrorOf | None = None +) -> RunLabels: + """The ``RunLabels`` of an item's runs; ``passed`` is the kind's own per-run verdict.""" + return RunLabels(tuple(run_outcome(r, passed=passed(r), judge_error=judge_error) for r in run_results)) + + +def attach_labels(exc: BaseException, labels: RunLabels, gate: str | None) -> None: + """Set ``outcome`` (only when every run errored) and ``shadow`` on an exception a kind + raises by hand -- the counterpart of the two ``raise_agentic_failure`` arguments.""" + outcome = labels.outcome_record + if outcome is not None: + exc.outcome = outcome # ty: ignore[unresolved-attribute] + exc.shadow = labels.shadow(gate) # ty: ignore[unresolved-attribute] + + +def gate_head( + gate: str | None, labels: RunLabels, runs_passed: int, failed_runs: Sequence[dict], runs_ungraded: int = 0 +) -> str: + """The start of a kind's failure message: the gate result, then every failing run. + + An item whose every run errored says so first, so a reader sees at once that the agent + was never scored. The gate note itself is the one the item was judged by. + """ + runs_total = len(labels.outcomes) + note = gate_failure_note(gate, runs_passed, runs_total, runs_ungraded) + item = labels.item_outcome + if item is not None: + note = f"Harness error, no run was scored ({item.reason}). {note}" + return failure_head(note, failed_runs, runs_total) + + +def log_eval_error_score(ctx: Any, trace_id: Any, outcome: Outcome) -> None: + """Mark one ERROR run in Langfuse: ``eval_error`` = 1, its comment the reason. + + An extra score only. The run's pass, gate, check and quality scores are written as + always, so nothing read from Langfuse changes; the marker says why the run failed. + """ + ctx.score(trace_id, name=EVAL_ERROR_SCORE, value=1, data_type="NUMERIC", comment=outcome.reason) + + +def mark_unscored_run(ctx: Any, run_idx: int, conversation_id: str, error: Outcome | None) -> None: + """Langfuse for a run that gets no verdict scores (its judge or its request failed). + + A harness ERROR (``error``, from ``RunLabels.errored_run``) gets its run root with the + ``eval_error`` marker; any other unscored run only keeps its root, as before. + """ + if error is None: + ctx.export_unscored(run_idx, conversation_id) + return + with ctx.observe(ctx.trace(conversation_id), run_idx, conversation_id=conversation_id) as tid: + log_eval_error_score(ctx, tid, error) def agentic_detail( @@ -60,6 +213,8 @@ def raise_agentic_failure( timings: PhaseTimings | None = None, failed_runs: list[dict] | None = None, run_latencies_s: list[float] | None = None, + outcome: dict[str, Any] | None = None, + shadow: dict[str, Any] | None = None, ) -> NoReturn: """Build ``exception_cls(message)`` with every common field attached, and raise it. @@ -68,6 +223,10 @@ def raise_agentic_failure( "no verdict at all" branch raises a bare ``JudgeResponseError`` instead (not a subclass) does not go through this helper for that branch. + ``outcome`` is set only when every run errored (``RunLabels.outcome_record``), and + ``shadow`` is the phase-2 candidate verdict (``RunLabels.shadow``); both are labels a + runner reads, neither changes the failure. + ``timings`` stays optional: only the three kinds that track per-run ``PhaseTimings`` (general_question, metric_skill, dashboard_skill) pass one -- the rest keep their own ``PhaseTimings()`` zero default, same as before this helper existed. ``failed_runs`` @@ -88,6 +247,10 @@ def raise_agentic_failure( exc.failed_runs = failed_runs if run_latencies_s is not None: exc.run_latencies_s = run_latencies_s + if outcome is not None: + exc.outcome = outcome + if shadow is not None: + exc.shadow = shadow raise exc diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py index 3f9d3028e..a549f6fef 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py @@ -217,9 +217,10 @@ def export_unscored(self, run_idx: int, conversation_id: str) -> None: with self.observe(self.trace(conversation_id), run_idx, conversation_id=conversation_id): pass - def score(self, trace_id: Any, *, name: str, value: Any, data_type: str) -> None: + def score(self, trace_id: Any, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: """Write one score, swallowing Langfuse failures the way ``score_safe`` always has.""" - self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type) + extra = {"comment": comment} if comment is not None else {} + self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type, **extra) def quality(self, trace_id: Any, *, strict_checks: dict, latency_sec: Any, cost_usd: Any) -> None: """Write the derived quality/value scores for one run.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index f1f532af3..ee6a439cd 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -15,15 +15,23 @@ from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1021,7 +1029,7 @@ def _run_once(conv_id: str) -> AlertRunResult: partial_alert_id, _, partial_tool_called = _extract_alert_call(partial.tool_call_events or []) if partial_tool_called: alert_id_to_delete = partial_alert_id - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break # `turns_used` counts attempts (set above, so a failed send still shows the # loop reached this iteration); `turns`/`steps` count completed work, so they @@ -1164,6 +1172,11 @@ def _run_detail(run: AlertRunResult) -> dict: } +def _failed_checks(run: AlertRunResult) -> list[dict]: + """The checks this run failed, by name. The arguments the run sent are in its detail.""" + return failed_check_records(FailedCheck(name) for name, ok in run.eval.checks().items() if not ok) + + def evaluate_agentic_alert_skill( host: str, token: str, @@ -1220,6 +1233,9 @@ def evaluate_agentic_alert_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.eval.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -1231,6 +1247,8 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.eval.checks() with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.errored_run(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -1288,10 +1306,11 @@ def _write_scores(ctx: RunTraceContext) -> None: summary.run_results, passed=lambda r: r.eval.strict_pass, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs) raise_agentic_failure( AlertSkillAssertionError, f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " @@ -1302,6 +1321,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"attributes_correct={ev.attributes_correct}, " f"granularity_correct={ev.granularity_correct}. " f"Actual args: {best.actual_alert_arguments}", + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py index 25f2281a0..6897d68d1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py @@ -36,12 +36,19 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -421,7 +428,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) viz_args, execute_result = _extract_anomaly_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -583,6 +590,9 @@ def evaluate_agentic_anomaly_detection( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -613,6 +623,8 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + if (run_error := run_labels.errored_run(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -660,7 +672,7 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs) message = ( f"Anomaly detection assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " @@ -672,6 +684,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( AnomalyDetectionAssertionError, message, + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 92f106340..5d227c9e8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -56,7 +56,13 @@ summarize_visualizations, ) from gooddata_eval.core.agentic._gate import log_gate_scores -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1038,7 +1044,7 @@ def run_agentic_conversation( conversation_reasoning_step_events.extend(partial.reasoning_step_events or []) reasoning_steps.extend(partial.reasoning_steps or []) response_id = partial.response_id or response_id - turn_exit = LoopExit.CHAT_ERROR + turn_exit = chat_failure_exit(exc) break final_result = chat_result total_steps += chat_result.reasoning_step_count @@ -1255,7 +1261,7 @@ def run_agentic_conversation( # A chat fault used to escape the whole run, so reaching here did mean no # error. Now that it is caught and recorded, this has to read it back -- # otherwise a turn whose chat call failed reports no_error=True. - no_error=turn_exit is not LoopExit.CHAT_ERROR, + no_error=turn_exit not in CHAT_FAILURE_EXITS, activated_skills=declared or [], active_skills=sorted(active_skills), clarification_turns_used=clarification_turns, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py index b7f2739f9..8449830e0 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -13,16 +13,24 @@ from gooddata_eval.core.agentic._clarification import all_questions from gooddata_eval.core.agentic._conversation_context import CONFIRMATION_REPLY, asks_in_text, classify_reply, clip from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._filters import describe_selection, same_unordered_values, selection_of from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -2302,7 +2310,7 @@ def _execute_single_dashboard_run( all_tool_call_events.extend(partial.tool_call_events or []) all_reasoning_step_events.extend(partial.reasoning_step_events or []) steps += partial.reasoning_step_count - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -2454,10 +2462,17 @@ def _run_detail(run: DashboardRunResult) -> dict[str, Any]: **run.evaluation.diagnostics, "failures": run.evaluation.failures, "notes": run.evaluation.notes, + # Why the loop stopped -- see LoopExit. + "exit_reason": run.exit_reason.value, "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), } +def _failed_checks(run: DashboardRunResult) -> list[dict]: + """The checks this run failed, by name. What each one found is in the run's ``failures``.""" + return failed_check_records(FailedCheck(name) for name, ok in run.evaluation.strict_checks.items() if not ok) + + def run_agentic_dashboard_skill( host: str, token: str, @@ -2580,6 +2595,9 @@ def evaluate_agentic_dashboard_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -2591,6 +2609,8 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.evaluation.strict_checks with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.errored_run(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") for name, value in run.evaluation.diagnostics.items(): @@ -2642,10 +2662,11 @@ def _write_scores(ctx: RunTraceContext) -> None: summary.run_results, passed=lambda r: r.evaluation.strict_pass, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs) # Two failures wear the same missing skill, and the advice differs. If the other # dashboard skill was activated, the flag that registers both is plainly on and the # model simply routed the wrong way. If neither was, the flag is off or the agent has @@ -2669,6 +2690,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Checks: {best.evaluation.strict_checks}. " f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." + (f" Notes (not scored): {notes}." if notes else ""), + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index 1b69e2175..fd8d46377 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -20,11 +20,11 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import attach_labels, gate_head, label_runs, mark_unscored_run from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1716,6 +1716,12 @@ class DocumentSkillAssertionError(AgenticAssertionError): """Raised when a document-skill evaluation fails.""" +def _judge_error_of(run: DocumentRunResult) -> str | None: + """The judge's own failure, for the harness label. ``run.judge_error`` also reports a failed + takeaway request (``fill_error``), which is on the product path and stays a FAIL.""" + return run.evaluation.judge_error + + def evaluate_agentic_document_skill( host: str, token: str, @@ -1795,6 +1801,9 @@ def evaluate_agentic_document_skill( document_format=document_format, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, judge_error=_judge_error_of) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -1806,7 +1815,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.ungraded: # No verdict decided this run: its other scores would publish a pass the gate # never counted. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.errored_run(run_idx)) continue pt = ctx.trace(run.conversation_id) strict_checks = run.evaluation.strict_checks @@ -1866,6 +1875,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) exc_judge.timings = item_timings exc_judge.run_latencies_s = [r.run_latency_s for r in summary.run_results] + attach_labels(exc_judge, run_labels, gate) raise exc_judge runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) @@ -1896,10 +1906,15 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: } # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. - failed_runs = build_failed_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, detail=_run_detail) + failed_runs = build_failed_runs( + summary.run_results, + passed=lambda r: r.evaluation.strict_pass, + detail=_run_detail, + judge_error=_judge_error_of, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) skill_note = "" if not best.evaluation.wire_names_current: skill_note = " gen-ai still sends the Report names, so it predates the Document rename." @@ -1923,6 +1938,7 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: exc.runs_effective = runs_effective exc.failed_runs = failed_runs exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] + attach_labels(exc, run_labels, gate) raise exc return AgenticEvalOutcome( runs_passed=runs_passed, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 907f0567a..af50b3cf6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -10,12 +10,19 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + attach_labels, + gate_head, + label_runs, + mark_unscored_run, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -339,6 +346,9 @@ def evaluate_agentic_general_question( document_format=document_format, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.judge_error is None and r.passed) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -350,7 +360,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.judge_error is not None: # No verdict for this run: float(run.passed) would write a 0 the judge # never returned. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.errored_run(run_idx)) continue pt = ctx.trace(run.conversation_id) with ctx.observe( @@ -431,13 +441,16 @@ def _write_scores(ctx: RunTraceContext) -> None: error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, run_latency_s(best.timings) ) error.timings = item_timings + attach_labels(error, run_labels, gate) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) raise_agentic_failure( GeneralQuestionAssertionError, f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index f9e281c2d..37e46fd37 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -10,12 +10,19 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + attach_labels, + gate_head, + label_runs, + mark_unscored_run, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -299,6 +306,9 @@ def evaluate_agentic_guardrail( user_context=user_context, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.judge_error is None and r.passed) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -310,7 +320,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.judge_error is not None: # No verdict, so nothing truthful to publish: float(run.passed) would # write a 0 the judge never returned. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.errored_run(run_idx)) continue pt = ctx.trace(run.conversation_id) with ctx.observe( @@ -386,13 +396,16 @@ def _write_scores(ctx: RunTraceContext) -> None: _attach_diagnostics( error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, best.run_latency_s ) + attach_labels(error, run_labels, gate) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) raise_agentic_failure( GuardrailAssertionError, f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 683f8d42d..6aea15f50 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -14,12 +14,19 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -329,7 +336,7 @@ def _accumulate(result: ChatResult) -> None: if create_args is not None: turn_wall_clock_sec = partial.turn_wall_clock_sec turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break turns += 1 steps += chat_result.reasoning_step_count @@ -499,6 +506,9 @@ def evaluate_agentic_kda_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -525,6 +535,8 @@ def _write_scores(ctx: RunTraceContext) -> None: "[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, turn_wall_clock_sec ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.errored_run(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN") @@ -590,7 +602,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs) message = ( f"KDA skill assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, " @@ -601,6 +613,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( KdaSkillAssertionError, message, + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index f662b205b..a00586702 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -13,15 +13,23 @@ from gooddata_sdk import GoodDataSdk from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -352,7 +360,7 @@ def _execute_single_metric_run( for metric_id in _extract_created_metric_ids(partial.tool_call_events or []): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -562,6 +570,9 @@ def evaluate_agentic_metric_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.metric_created and r.maql_correct) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -577,6 +588,8 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=run.conversation_id, output={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, ) as tid: + if (run_error := run_labels.errored_run(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -631,6 +644,21 @@ def _run_detail(run: MetricRunResult) -> dict: **timeline_detail(run.tool_call_events, run.reasoning_step_events), } + def _failed_checks(run: MetricRunResult) -> list[dict]: + """The checks this run failed; the MAQL pair is the whole diagnosis for this kind.""" + checks: list[FailedCheck] = [] + if not run.metric_created: + checks.append(FailedCheck("metric_created", expected=True, actual=False)) + if not run.maql_correct: + checks.append( + FailedCheck( + "maql_correct", + expected=[c.get("maql", "") for c in expected_outputs_list], + actual=run.actual_maql, + ) + ) + return failed_check_records(checks) + detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, @@ -650,10 +678,11 @@ def _run_detail(run: MetricRunResult) -> dict: summary.run_results, passed=lambda r: r.metric_created and r.maql_correct, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) raise_agentic_failure( MetricSkillAssertionError, @@ -661,6 +690,8 @@ def _run_detail(run: MetricRunResult) -> dict: f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " f"Actual MAQL: {best.actual_maql}.", + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index b06821452..78bbb04ac 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -12,15 +12,24 @@ from dataclasses import dataclass, field, replace from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -232,21 +241,21 @@ def _execute_single_run( current_result = client.send_message(conversation_id, question) except ChatError as exc: # The opening request, so there is no partial conversation to evaluate: fall through - # to the empty-result path with CHAT_ERROR recorded. Raising here would discard every + # to the empty-result path with its exit recorded. Raising here would discard every # K-run already completed along with any exit_reason. print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}") current_result = getattr(exc, "partial_result", None) or ChatResult() - # The loop that collects events does not run after CHAT_ERROR, so a partial result's + # The loop that collects events does not run after a failed opening request, so a partial result's # chart executions are added here. all_tool_call_events.extend(current_result.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) # Counted here, not at the top of the loop: this request is sent unconditionally, so # with max_iterations=0 the loop body never runs and total_turns would report 0 turns # for a conversation the agent did receive. The loop's own increment is skipped on its # first pass to compensate, keeping total_turns == number of send_message calls. total_turns += 1 - for iteration in range(max_iterations if exit_reason is not LoopExit.CHAT_ERROR else 0): + for iteration in range(max_iterations if exit_reason not in CHAT_FAILURE_EXITS else 0): if iteration: total_turns += 1 total_steps += current_result.reasoning_step_count @@ -297,7 +306,7 @@ def _execute_single_run( # The loop adds a turn's events at its next pass, which a break skips. The # partial result is scored, so its chart executions count too. all_tool_call_events.extend(partial.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break skill_activated = _check_visualization_skill_activated(all_tool_call_events) @@ -316,7 +325,7 @@ def _execute_single_run( actual_output, all_tool_call_events, answer_text, - interrupted=exit_reason is LoopExit.CHAT_ERROR, + interrupted=exit_reason in CHAT_FAILURE_EXITS, ) return RunResult( @@ -445,6 +454,74 @@ def _run_detail(run: RunResult) -> dict: } +def _failed_checks(run: RunResult) -> list[dict]: + """The checks this run failed, by their ``detail`` names, with expected/actual where known. + + Follows ``EvaluationResult.strict_pass`` term by term, so the list is empty exactly when + the run passed. + """ + ev = run.eval_result + checks: list[FailedCheck] = [] + if ev.scored_without_chart: + if ev.visualization_created: + actual_type = run.actual_output.type if run.actual_output else None + checks.append(FailedCheck("no_chart", expected="no chart", actual=actual_type)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + return failed_check_records(checks) + if not ev.visualization_created: + checks.append(FailedCheck("visualization_created", expected=True, actual=False)) + if not ev.cross_ref_valid: + checks.append(FailedCheck("cross_ref_valid", actual=ev.cross_ref_errors)) + if not ev.metrics_correct: + if ev.expected_metric_uris == ev.actual_metric_uris and ev.metric_layout_problems: + # The metric set matched; the order or axis did not, and only these say which. + checks.append(FailedCheck("metrics_correct", label="layout", actual=ev.metric_layout_problems)) + else: + checks.append( + FailedCheck( + "metrics_correct", expected=sorted(ev.expected_metric_uris), actual=sorted(ev.actual_metric_uris) + ) + ) + if not ev.dimensions_correct: + checks.append( + FailedCheck("dimensions_correct", expected=sorted(ev.expected_dim_uris), actual=sorted(ev.actual_dim_uris)) + ) + if not ev.filters_correct: + differing = [ + FailedCheck( + "filters_correct", + label=category, + expected=ev.expected_filters.get(category, []), + actual=ev.actual_filters.get(category, []), + ) + for category in sorted(set(ev.expected_filters) | set(ev.actual_filters)) + if ev.expected_filters.get(category, []) != ev.actual_filters.get(category, []) + ] + checks.extend(differing or [FailedCheck("filters_correct")]) + if not ev.sorts_correct: + checks.append(FailedCheck("sorts_correct", expected=ev.expected_sorts, actual=ev.actual_sorts)) + if not ev.viz_type_hard: + checks.append( + FailedCheck( + "viz_type_hard", + expected=run.best_expected.type if run.best_expected else None, + actual=run.actual_output.type if run.actual_output else None, + ) + ) + if ev.requires_execution and not ev.executed: + checks.append(FailedCheck("executed", expected=True, actual=False)) + if ev.buckets_correct is False: + checks.append(FailedCheck("buckets_correct", expected=ev.expected_buckets, actual=ev.actual_buckets)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + if ev.format_correct is False: + checks.append(FailedCheck("format_correct", actual=ev.format_problems)) + if ev.part_present is False: + checks.append(FailedCheck("part_present", expected=True, actual=False)) + return failed_check_records(checks) + + def _strict_checks(ev: EvaluationResult) -> dict[str, bool]: """The scores one run is gated on, by their Langfuse names. @@ -579,6 +656,9 @@ def evaluate_agentic_visualization( options=options, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.eval_result.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -592,6 +672,8 @@ def _write_scores(ctx: RunTraceContext) -> None: ev = run.eval_result strict_checks = _strict_checks(ev) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.errored_run(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): # Scored below on every item, gating or not. if score_name != "assertion-vis-executed": @@ -664,10 +746,11 @@ def _write_scores(ctx: RunTraceContext) -> None: summary.run_results, passed=lambda r: r.eval_result.strict_pass, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs) n = len(expected_outputs) candidate_note = f" (closest of {n} candidates)" if n > 1 else "" cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else "" @@ -711,6 +794,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"{_options_lines(ev, best.answer_text)}" f" Stated Value Matches : {ev.stated_value_matches} (reported only)\n" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py index de124ceed..a0ad72f8d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py @@ -30,12 +30,19 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + gate_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -378,7 +385,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) create_args, execute_result = _extract_what_if_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -544,6 +551,9 @@ def evaluate_agentic_what_if( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -576,6 +586,8 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + if (run_error := run_labels.errored_run(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -624,7 +636,7 @@ def _write_scores(ctx: RunTraceContext) -> None: runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + gate_note = gate_head(gate, run_labels, runs_passed, failed_runs) message = ( f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " @@ -637,6 +649,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( WhatIfAssertionError, message, + outcome=run_labels.outcome_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 094cee08e..d3ef0364b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -28,6 +28,11 @@ class LoopExit(str, Enum): This does not change any verdict -- an exhausted run still fails. It makes the three cases countable, so "is the turn budget too tight" becomes a question the data can answer rather than one that has to be argued. + + Which members label a run ERROR (the harness failed) rather than FAIL is decided in one + place, ``core/outcome.py``. The label decides no verdict. Only SIMULATED_USER_FAILED is an + ERROR: CHAT_ERROR, AGENT_TIMEOUT and TURN_INCOMPLETE are on the product path, and a + customer hits them too. """ SUCCESS = "success" @@ -48,16 +53,27 @@ class LoopExit(str, Enum): SIMULATED_USER_FAILED = "simulated_user_failed" """The harness's own simulated-user model failed to produce a follow-up. A harness-side - fault, not an agent one -- reporting should treat it as an error, not a scored failure.""" + fault, not an agent one, so the run is labelled ERROR (``core/outcome.py``).""" CHAT_ERROR = "chat_error" - """The chat call itself raised mid-conversation. Like SIMULATED_USER_FAILED this is - infrastructure rather than agent capability, but it comes from the GoodData side.""" + """The chat call itself raised mid-conversation and was not one of the agent's own + failures below: a gen-ai error event, a broken stream, or a 5xx still failing after its + retries. It is on the product path -- a customer would see it too -- so the run is a + FAIL, not a harness ERROR.""" + + AGENT_TIMEOUT = "agent_timeout" + """The agent exceeded the per-turn or per-item wall-clock budget (``TurnTimeoutError``). + The agent's own failure to answer in time, so the run is a FAIL.""" + + TURN_INCOMPLETE = "turn_incomplete" + """The agent ended its turn without a final answer, e.g. at its own iteration limit + (``TurnIncompleteError``). The agent's own failure, so the run is a FAIL.""" NOT_RUN = "not_run" """The loop never started -- e.g. an agentic_conversation turn whose $ref to an earlier turn's output could not be resolved, so the turn was skipped before any message was - sent. Distinct from every other member, which all imply at least one turn happened.""" + sent. Distinct from every other member, which all imply at least one turn happened. A + FAIL: its cause is usually the earlier turn's failure, which the item already reports.""" class AacQueryField(BaseModel): @@ -441,6 +457,12 @@ class AgenticAssertionError(AssertionError): failed_runs: list[dict] run_latencies_s: list[float] best_run_latency_s: float | None + outcome: dict + """``{"status": "error", "reason": ...}``, set only when every run of the item errored + (``core/outcome.py``): the item is a harness error, not an agent failure. A runner labels + it; it stays a failure.""" + shadow: dict + """The phase-2 candidate verdict (``core.outcome.shadow_verdict``). Recorded, never used.""" class AgenticEvalOutcome(BaseModel): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py new file mode 100644 index 000000000..539ca5267 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py @@ -0,0 +1,143 @@ +# (C) 2026 GoodData Corporation +"""One label per run: PASS, FAIL, or ERROR when the test harness, not the agent, failed. + +A run can end in a way that is the harness's fault: the simulated user's own model call +fails, or the judge cannot grade the answer. Scored as usual, such a run reads as an agent +failure. ``classify_run`` names the difference, in one place for every kind. + +The labels decide no verdict. A run labelled ERROR still counts as a run that did not pass, +so every gate, pass rate and trend is what it was before labels existed. The labels only +tell a reader why a run did not pass (QA-30213, phase 1). ``shadow_verdict`` records what a +candidate rule that leaves ERROR runs out of the gate would have given, so that rule can be +judged on real nights before anyone adopts it (QA-30296). + +The rule is fail-closed. A run is ERROR only for a cause on ``_ERROR_EXITS`` or a judge +failure: the agent never had a fair chance, and a customer would not have hit the cause. +Anything else, including a chat API error and an unknown exit, is FAIL. +""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from gooddata_eval.core.config import normalize_gate +from gooddata_eval.core.models import LoopExit + +__all__ = [ + "EVAL_ERROR_SCORE", + "JUDGE_FAILED", + "Outcome", + "OutcomeStatus", + "classify_run", + "item_outcome", + "shadow_verdict", +] + + +class OutcomeStatus(str, Enum): + """What one run, or one item, amounts to.""" + + PASS = "pass" + FAIL = "fail" + ERROR = "error" + """The test harness failed, so the run says nothing about the agent.""" + + +@dataclass(frozen=True) +class Outcome: + """A status and a machine-readable reason, e.g. ``simulated_user_failed``.""" + + status: OutcomeStatus + reason: str | None = None + + @classmethod + def passed(cls) -> Outcome: + return cls(OutcomeStatus.PASS) + + @classmethod + def failed(cls, reason: str | None = None) -> Outcome: + return cls(OutcomeStatus.FAIL, reason) + + @classmethod + def error(cls, reason: str) -> Outcome: + return cls(OutcomeStatus.ERROR, reason) + + @property + def is_error(self) -> bool: + return self.status is OutcomeStatus.ERROR + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into ``failed_runs[].outcome`` and onto the exception.""" + return {"status": self.status.value, "reason": self.reason} + + +EVAL_ERROR_SCORE = "eval_error" +"""The extra Langfuse score (NUMERIC, 1) an ERROR run gets, its comment the reason. The +run's other scores are written as always.""" + +JUDGE_FAILED = "judge_failed" +"""The reason of a run the judge or scorer could not grade.""" + +# The only loop exits that make a run an ERROR. A chat API error stays a FAIL: it is on the +# product path, and a customer would hit it too. +_ERROR_EXITS: frozenset[LoopExit] = frozenset({LoopExit.SIMULATED_USER_FAILED}) + + +def classify_run(exit_reason: LoopExit | str | None, *, passed: bool, judge_error: str | None = None) -> Outcome: + """The label of one run, from whether it passed, why its loop stopped and its judge. + + A run that passed is PASS whatever its exit: its checks held on what the agent produced. + A run whose judge failed is ERROR (``judge_failed``), and so is a run whose exit is on + ``_ERROR_EXITS``. Every other run is FAIL, its reason the exit (or None for a kind whose + loop records none). An exit string that is not a ``LoopExit`` value is kept as the reason + of a FAIL rather than raising: an unknown cause is never an error. + """ + if passed: + return Outcome.passed() + if judge_error is not None: + return Outcome.error(JUDGE_FAILED) + loop_exit: LoopExit | None + if isinstance(exit_reason, LoopExit) or exit_reason is None: + loop_exit = exit_reason + else: + try: + loop_exit = LoopExit(exit_reason) + except ValueError: + return Outcome.failed(exit_reason) + if loop_exit in _ERROR_EXITS: + return Outcome.error(loop_exit.value) + return Outcome.failed(loop_exit.value if loop_exit is not None else None) + + +def item_outcome(outcomes: Sequence[Outcome]) -> Outcome | None: + """ERROR when every run of the item is ERROR, with the most frequent reason; else None. + + None means the item keeps the verdict its gate gave it. An item with some ERROR runs is + not an ERROR: its other runs were scored, and its message names the ERROR runs. + """ + if not outcomes or not all(o.is_error for o in outcomes): + return None + reasons = Counter(o.reason or "unknown" for o in outcomes) + return Outcome.error(max(sorted(reasons), key=lambda reason: reasons[reason])) + + +def shadow_verdict(outcomes: Sequence[Outcome], gate: str | None) -> dict[str, Any]: + """What the candidate phase-2 rule would give this item. Recorded, never used. + + The rule leaves ERROR runs out of K. With fewer than ceil(K/2) scored runs left the item + is ``error``; otherwise ``gate`` is applied to the scored runs. + """ + scored = [o for o in outcomes if not o.is_error] + passes = sum(1 for o in scored if o.status is OutcomeStatus.PASS) + if not scored or len(scored) < math.ceil(len(outcomes) / 2): + verdict = "error" + elif normalize_gate(gate) == "power": + verdict = "pass" if passes == len(scored) else "fail" + else: + verdict = "pass" if passes else "fail" + return {"verdict": verdict, "runs_scored": len(scored), "runs_total": len(outcomes)} diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 09cfab387..9513102e3 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -26,7 +26,7 @@ render_turn_for_simulated_user, run_agentic_alert_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _DATE_FILTER = { @@ -1932,3 +1932,21 @@ def test_an_alert_created_before_the_stream_broke_is_still_cleaned_up(): assert mock_delete.call_count == 1 assert mock_delete.call_args.args[-1] == "alert-1" + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + ], +) +def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = exc + + detail = _run_alert(mock_client, max_iterations=3) + + assert detail["exit_reason"] == exit_value + assert detail["alert_created"] is False diff --git a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py index 5c1e25d8b..4bdfe156b 100644 --- a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py +++ b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py @@ -16,6 +16,7 @@ evaluate_agentic_anomaly_detection, run_agentic_anomaly_detection, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.anomaly_detection" @@ -431,7 +432,7 @@ def trace(self, _conversation_id): def observe(self, _trace, _run_idx, *, conversation_id=None): yield "trace-id" - def score(self, _tid, *, name, value, data_type): + def score(self, _tid, *, name, value, data_type, comment=None): self.scores[name] = value def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): @@ -539,3 +540,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index ca0f2dc2d..f08b0d63f 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -53,7 +53,7 @@ run_agentic_conversation, ) from gooddata_eval.core.chat.render import render_answer_text -from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.chat.sse_client import ChatError as ChatError_ from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent @@ -1209,6 +1209,32 @@ def test_a_partial_result_joins_the_shared_timeline_like_any_other_turn(): assert result.total_steps == 5 +def test_an_agent_timeout_ends_its_turn_with_its_own_exit_and_still_reads_as_an_error(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [ + _metric_turn_result([_skills_tc("metric"), _create_metric_tc("m1")]), + TurnTimeoutError("agent exceeded the 60s turn budget"), + ] + + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + ): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=_two_metric_turn_fixture(), + ) + + assert result.turn_results[0].exit_reason is LoopExit.SUCCESS + assert result.turn_results[1].exit_reason is LoopExit.AGENT_TIMEOUT + # The turn's no_error reads exactly as it did when a timeout was recorded as CHAT_ERROR. + assert result.turn_results[1].no_error is False + assert result.turn_results[1].output_present is False + + def test_a_non_chat_exception_still_propagates(): """Only chat faults are absorbed. A programming error must not be relabelled CHAT_ERROR.""" mock_client = MagicMock() diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py index 9b6075b9f..d053be358 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -20,7 +20,7 @@ evaluate_dashboard_response, run_agentic_dashboard_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent # Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of @@ -2947,6 +2947,22 @@ def test_a_caller_supplied_conversation_is_not_deleted(self): client.create_conversation.assert_not_called() client.delete_conversation.assert_not_called() + @pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], + ) + def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(self, exc, loop_exit) -> None: + client = MagicMock() + client.send_message.side_effect = exc + + summary = _run_with(client, _DC05_EXPECTED) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + def test_a_chat_error_is_recorded_rather_than_left_to_escape(self) -> None: """It used to escape run_agentic_dashboard_skill, taking every completed K-run with it: no DashboardRunResult, no exit_reason, just a bare error on the item. Recorded diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index e1a0c0f2d..78d79410a 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -1733,6 +1733,7 @@ def __init__(self) -> None: self.quality_checks: dict[str, bool] = {} self.observed: list[str] = [] self.unscored: list[str] = [] + self.comments: dict[str, str] = {} def export_unscored(self, _run_idx: int, conversation_id: str) -> None: self.unscored.append(conversation_id) @@ -1741,13 +1742,17 @@ def trace(self, _conversation_id: str) -> None: return None @contextmanager - def observe(self, _trace: None, _run_idx: int, *, conversation_id: str, output: dict) -> Iterator[str]: + def observe( + self, _trace: None, _run_idx: int, *, conversation_id: str, output: dict | None = None + ) -> Iterator[str]: self.observed.append(conversation_id) yield "trace-id" - def score(self, _tid: str, *, name: str, value: float, data_type: str) -> None: + def score(self, _tid: str, *, name: str, value: float, data_type: str, comment: str | None = None) -> None: self.scores[name] = value self.score_types[name] = data_type + if comment is not None: + self.comments[name] = comment def quality( self, _tid: str, *, strict_checks: dict[str, bool], latency_sec: float | None, cost_usd: float | None @@ -1991,10 +1996,12 @@ def test_an_ungraded_run_keeps_pass_at_k_but_not_pass_power_k(monkeypatch: pytes assert summary.best.judge_error is None -def test_an_ungraded_run_writes_no_scores(monkeypatch: pytest.MonkeyPatch) -> None: +def test_a_run_the_judge_could_not_grade_gets_only_the_harness_marker(monkeypatch: pytest.MonkeyPatch) -> None: ctx = _scored(monkeypatch, {"narrative": _NARRATIVE}, _narrative_turns(), k=2, judge=_SequenceJudge()) - assert ctx.observed == ["conv-2"] - assert ctx.unscored == ["conv-1"] + # conv-1 has no verdict: its root carries eval_error alone; conv-2 is scored as before. + assert ctx.observed == ["conv-1", "conv-2"] + assert ctx.unscored == [] + assert (ctx.scores["eval_error"], ctx.comments["eval_error"]) == (1, "judge_failed") assert ctx.scores["document_narrative_judged"] == 1.0 assert ctx.scores["document_summaries_present"] == 1.0 diff --git a/packages/gooddata-eval/tests/test_agentic_failure_message.py b/packages/gooddata-eval/tests/test_agentic_failure_message.py new file mode 100644 index 000000000..22907fd7c --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_failure_message.py @@ -0,0 +1,356 @@ +# (C) 2026 GoodData Corporation +"""The failing-runs head of an agentic failure message, and the per-run records it reads.""" + +import importlib +import inspect +from dataclasses import dataclass, field +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import metric_skill as metric_skill_mod +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import ( + FailedCheck, + failed_check_records, + failing_runs_note, + failure_head, +) +from gooddata_eval.core.agentic.metric_skill import MetricSkillAssertionError, evaluate_agentic_metric_skill +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit +from gooddata_eval.core.outcome import classify_run + +# ── the renderer ───────────────────────────────────────────────────────────── + + +def test_a_failed_check_records_only_what_the_kind_knows() -> None: + assert FailedCheck("metric_correct").as_record() == {"name": "metric_correct"} + assert FailedCheck("filters_correct", label="date", expected=[], actual=["MONTH -11..0"]).as_record() == { + "name": "filters_correct", + "label": "date", + "expected": [], + "actual": ["MONTH -11..0"], + } + # None is a real expected value, not "unknown". + assert FailedCheck("x", expected=None).as_record() == {"name": "x", "expected": None} + + +def test_the_note_lists_each_failing_run_with_its_exit_and_failing_checks_only() -> None: + failed_runs = [ + { + "run_index": 2, + "exit_reason": "success", + "failed_checks": failed_check_records( + [FailedCheck("filters_correct", label="date", expected=[], actual="MONTH -11..0")] + ), + }, + {"run_index": 3, "exit_reason": "budget_exhausted", "failed_checks": [{"name": "metric_correct"}]}, + ] + + note = failing_runs_note(failed_runs, 3) + + assert note == ( + "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual MONTH -11..0]; " + "run 3/3 FAIL (loop exit: budget_exhausted): metric_correct" + ) + + +def test_the_head_starts_with_the_gate_note() -> None: + gate_note = "Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure." + failed_runs = [{"run_index": 2, "exit_reason": "success", "failed_checks": [{"name": "metric_correct"}]}] + + head = failure_head(gate_note, failed_runs, 3) + + assert head.startswith(gate_note) + assert head.endswith("Failing runs: run 2/3 FAIL (loop exit: success): metric_correct.") + + +def test_the_head_is_the_gate_note_alone_without_failing_runs() -> None: + assert failure_head("Gate pass@2 failed: 0/2 runs passed.", [], 2) == "Gate pass@2 failed: 0/2 runs passed." + + +def test_a_run_without_recorded_checks_says_so_and_a_kind_without_the_key_says_nothing() -> None: + assert failing_runs_note([{"run_index": 1, "exit_reason": None, "failed_checks": []}], 1) == ( + "run 1/1 FAIL: no failing check recorded" + ) + assert failing_runs_note([{"run_index": 1, "exit_reason": "agent_silent"}], 1) == ( + "run 1/1 FAIL (loop exit: agent_silent)" + ) + + +def test_an_errored_run_is_labelled_with_its_reason_and_lists_no_checks() -> None: + record = { + "run_index": 1, + "exit_reason": "simulated_user_failed", + "outcome": {"status": "error", "reason": "simulated_user_failed"}, + "failed_checks": [{"name": "metric_created"}], + } + assert failing_runs_note([record], 3) == "run 1/3 ERROR (simulated_user_failed)" + + +def test_a_long_value_is_cut_to_one_short_line() -> None: + record = {"run_index": 1, "failed_checks": [{"name": "maql_correct", "actual": "SELECT\n" + "x" * 500}]} + note = failing_runs_note([record], 1) + assert "\n" not in note + assert note.endswith("...]") + assert len(note) < 250 + + +# ── the records ────────────────────────────────────────────────────────────── + + +@dataclass +class _Run: + ok: bool + exit_reason: LoopExit = LoopExit.SUCCESS + conversation_id: str = "c" + tool_call_events: list = field(default_factory=list) + + +def test_every_failing_record_carries_its_exit_reason_and_failed_checks() -> None: + runs = [_Run(True), _Run(False, LoopExit.BUDGET_EXHAUSTED)] + + records = build_failed_runs( + runs, + passed=lambda r: r.ok, + detail=lambda r: {}, + failed_checks=lambda r: [{"name": "metric_correct"}], + ) + + assert len(records) == 1 + assert records[0]["run_index"] == 2 + assert records[0]["exit_reason"] == "budget_exhausted" + assert records[0]["failed_checks"] == [{"name": "metric_correct"}] + + +def test_a_kind_that_names_no_checks_writes_no_failed_checks_key() -> None: + records = build_failed_runs([object()], passed=lambda r: False, detail=lambda r: {}) + assert records[0]["exit_reason"] is None + assert "failed_checks" not in records[0] + + +@dataclass +class _StrRun: + exit_reason: str = "chat_error" + + +def test_a_string_exit_reason_gets_the_same_outcome_as_its_enum() -> None: + # exit_reason and outcome must agree whether a kind records the member or its value. + for exit_reason in (LoopExit.SIMULATED_USER_FAILED, "simulated_user_failed"): + record = build_failed_runs([_StrRun(exit_reason)], passed=lambda r: False, detail=lambda r: {})[0] + assert record["exit_reason"] == "simulated_user_failed" + assert record["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + + +@pytest.mark.parametrize( + "exit_reason", [LoopExit.CHAT_ERROR, "chat_error", "budget_exhausted"], ids=["member", "value", "agent_exit"] +) +def test_a_chat_error_or_agent_exit_is_recorded_and_rendered_as_a_failure(exit_reason: LoopExit | str) -> None: + records = build_failed_runs( + [_StrRun(exit_reason)], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: [] + ) + + value = LoopExit(exit_reason).value + assert records[0]["exit_reason"] == value + assert records[0]["outcome"] == {"status": "fail", "reason": value} + assert failing_runs_note(records, 3) == f"run 1/3 FAIL (loop exit: {value}): no failing check recorded" + + +def test_a_simulated_user_failure_is_recorded_and_rendered_as_an_error() -> None: + records = build_failed_runs( + [_StrRun("simulated_user_failed")], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: [] + ) + + assert records[0]["outcome"]["status"] == "error" + assert failing_runs_note(records, 3) == "run 1/3 ERROR (simulated_user_failed)" + + +@dataclass +class _JudgedRun: + judge_error: str | None + exit_reason: str = "success" + + +def test_a_run_the_judge_could_not_grade_is_recorded_as_an_error() -> None: + (record,) = build_failed_runs([_JudgedRun("empty body twice")], passed=lambda r: False, detail=lambda r: {}) + assert record["outcome"] == {"status": "error", "reason": "judge_failed"} + + +def test_a_failed_check_record_keeps_short_values_and_caps_long_ones() -> None: + long_actual = {"filters": ["x" * 400]} + records = build_failed_runs( + [_StrRun("budget_exhausted")], + passed=lambda r: False, + detail=lambda r: {"actual": long_actual}, + failed_checks=lambda r: [ + {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]}, + {"name": "filters_correct", "label": "attribute", "expected": [], "actual": long_actual}, + ], + ) + + short, long_check = records[0]["failed_checks"] + assert short == {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]} + assert long_check["expected"] == [] + assert isinstance(long_check["actual"], str) + assert len(long_check["actual"]) == 160 + assert long_check["actual"].endswith("...") + # The run's detail keeps the whole value. + assert records[0]["detail"]["actual"] == long_actual + + +def test_a_fail_run_names_its_loop_exit_as_its_reason() -> None: + assert classify_run(LoopExit.SUCCESS, passed=False).reason == "success" + + +def test_a_metric_layout_failure_names_the_layout_problem_not_two_equal_sets() -> None: + ev = MagicMock( + scored_without_chart=False, + visualization_created=True, + cross_ref_valid=True, + metrics_correct=False, + expected_metric_uris={"metric/revenue"}, + actual_metric_uris={"metric/revenue"}, + metric_layout_problems=["metric/revenue is on the secondary axis"], + dimensions_correct=True, + filters_correct=True, + sorts_correct=True, + viz_type_hard=True, + requires_execution=False, + buckets_correct=None, + answer_correct=None, + format_correct=None, + part_present=None, + ) + run = MagicMock(eval_result=ev) + + assert agentic_visualization._failed_checks(run) == [ + {"name": "metrics_correct", "label": "layout", "actual": ["metric/revenue is on the secondary axis"]} + ] + + +# ── visualization: a partial gate failure names the run that failed ────────── + + +def _viz(filter_by: dict | None = None) -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": filter_by or {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn(filter_by: dict | None = None) -> ChatResult: + return ChatResult.model_validate( + {"createdVisualizations": {"objects": [_viz(filter_by)], "reasoning": ""}, "toolCallEvents": []} + ) + + +_LAST_YEAR = {"date": {"type": "date_filter", "using": "dataset/date", "granularity": "MONTH", "from": -11, "to": 0}} + + +def test_a_partial_visualization_failure_lists_the_failing_run_and_keeps_the_best_run_summary() -> None: + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [_viz_turn(), _viz_turn(_LAST_YEAR), _viz_turn()] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=3, + gate="power", + ) + + message = str(exc_info.value) + lines = message.splitlines() + gate_line = next(line for line in lines if line.startswith("Gate pass^3 failed")) + assert gate_line.startswith("Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure. Failing runs: ") + assert "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual " in gate_line + # Only the failing checks of the failing run, and no passing run. + assert "metrics_correct" not in gate_line + assert "run 1/3" not in gate_line + assert "run 3/3" not in gate_line + # The best-run summary that followed the gate note is still there, unchanged. + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + assert "Strict Check Summary:" in message + assert " Filters Correct : True\n" in message + + (record,) = exc_info.value.failed_runs + assert record["run_index"] == 2 + assert record["exit_reason"] == "success" + assert record["detail"]["exit_reason"] == "success" + assert [c["name"] for c in record["failed_checks"]] == ["filters_correct"] + assert record["failed_checks"][0]["label"] == "date" + assert record["failed_checks"][0]["expected"] == [] + + +# ── metric skill: same head, expected/actual MAQL of the failing run ───────── + + +def _metric_turn(maql: str) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": f'{{"data": {{"maql": "{maql}"}}}}', + } + ], + } + ) + + +def test_a_partial_metric_failure_lists_the_failing_run_with_its_maql() -> None: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = [_metric_turn("SELECT {metric/foo}"), _metric_turn("SELECT {metric/bar}")] + + with ( + patch.object(metric_skill_mod, "ChatClient", return_value=client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + gate="power", + ) + + message = str(exc_info.value) + assert message.startswith( + "Metric skill assertion failed. Gate pass^2 failed: 1/2 runs passed — unstable, not a clean failure. " + "Failing runs: run 2/2 FAIL (loop exit: success): " + 'maql_correct[expected ["SELECT {metric/foo}"], actual SELECT {metric/bar}].' + ) + # The best run's summary follows, as before. + assert "metric_created=True, maql_correct=True. Expected MAQL (candidates): 'SELECT {metric/foo}'." in message + (record,) = exc_info.value.failed_runs + assert record["exit_reason"] == "success" + assert record["failed_checks"] == [ + {"name": "maql_correct", "expected": ["SELECT {metric/foo}"], "actual": "SELECT {metric/bar}"} + ] + + +@pytest.mark.parametrize("module_name", ["visualization", "alert_skill", "metric_skill", "dashboard_skill"]) +def test_each_checked_kind_heads_its_message_through_the_shared_renderer(module_name: str) -> None: + source = inspect.getsource(importlib.import_module(f"gooddata_eval.core.agentic.{module_name}")) + assert "gate_head(gate, run_labels, runs_passed, failed_runs)" in source + assert "failed_checks=_failed_checks" in source diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 6a1690ebb..8f13a4093 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -16,8 +16,8 @@ evaluate_agentic_kda_skill, run_agentic_kda_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.models import ChatResult, LoopExit _EXPECTED = {"Measure": {"type": "metric", "id": "revenue"}} @@ -1296,3 +1296,29 @@ def _capture(_submit, _identity, **kwargs): assert scores["turns"] == 1 assert scores["steps"] == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("gen-ai returned 503"), LoopExit.CHAT_ERROR), + ], +) +def test_run_agentic_kda_skill_names_why_the_chat_call_failed(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + ) + + assert summary.best.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 5efcbb0bb..b8ebb0c63 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -22,7 +22,7 @@ generate_simulated_response, run_agentic_metric_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent from gooddata_eval.core.timing import TIMERS_ENV_VAR @@ -1098,3 +1098,81 @@ def call(mid: str, created_new: bool) -> ToolCallEvent: ) assert _extract_created_metric_ids([call("new", True), call("old", False)]) == ["new"] + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=3, + ) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + + +def test_a_chat_error_run_is_a_failure_scored_as_before(): + mock_client = _client() + mock_client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + } + ), + ] + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + _patched(mock_client), + patch("gooddata_eval.core.agentic.metric_skill.submit_trace_scoring", _capture), + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + run_ids = [MagicMock(name="run-1"), MagicMock(name="run-2")] + ctx.observe.return_value.__enter__.side_effect = run_ids + captured["write_scores"](ctx) + by_run = {id(t): [c.kwargs["name"] for c in ctx.score.call_args_list if c.args[0] is t] for t in run_ids} + + first, second = by_run[id(run_ids[0])], by_run[id(run_ids[1])] + # A chat API error is on the product path: a FAIL, with no harness marker. + assert "eval_error" not in first + assert "gate_passed" in first + assert "eval_error" not in second + assert "gate_passed" in second diff --git a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py index 83f31a277..44a4d4fba 100644 --- a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py +++ b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py @@ -345,3 +345,15 @@ def test_the_run_context_forwards_its_window_and_item_input_to_observe(): "item_input": "Show revenue by month", "output": {"passed": True}, } + + +def test_the_run_context_forwards_a_score_comment_only_when_given(): + lf = MagicMock() + ctx = RunTraceContext({}, lf, "client", "item-1", "base", True, {}, _WINDOW, "q") + + ctx.score("tid", name="gate_passed", value=True, data_type="BOOLEAN") + ctx.score("tid", name="eval_error", value=1, data_type="NUMERIC", comment="backend: the chat call failed") + + plain, commented = lf.score_safe.call_args_list + assert plain.kwargs == {"name": "gate_passed", "value": True, "data_type": "BOOLEAN"} + assert commented.kwargs["comment"] == "backend: the chat call failed" diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index 431c3f329..c50d2fddb 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -18,7 +18,7 @@ evaluate_agentic_visualization, run_agentic_visualization, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.evaluators.visualization import ExpectationOptions from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit @@ -822,3 +822,35 @@ def test_a_failed_request_with_a_cut_off_answer_fails_an_item_without_a_chart() assert result.exit_reason is LoopExit.CHAT_ERROR assert result.eval_result.strict_pass is False assert result.eval_result.answer_problems == ["the chat request failed before the answer ended"] + + +_AGENT_CHAT_FAILURES = [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), +] + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_the_opening_request(exc, loop_exit): + client = MagicMock() + client.send_message.side_effect = exc + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit + assert result.eval_result.visualization_created is False + # Like a chat error, the loop does not run after the opening failure. + assert client.send_message.call_count == 1 + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkeypatch, exc, loop_exit): + monkeypatch.setattr( + "gooddata_eval.core.agentic.visualization.generate_simulated_response", lambda *a, **k: "the revenue one" + ) + client = MagicMock() + client.send_message.side_effect = [_chat_clarification(), exc] + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_what_if.py b/packages/gooddata-eval/tests/test_agentic_what_if.py index 38b04ebf3..4ee0e4ab0 100644 --- a/packages/gooddata-eval/tests/test_agentic_what_if.py +++ b/packages/gooddata-eval/tests/test_agentic_what_if.py @@ -13,6 +13,7 @@ evaluate_agentic_what_if, run_agentic_what_if, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.what_if" @@ -385,7 +386,7 @@ def trace(self, _conversation_id): def observe(self, _trace, _run_idx, *, conversation_id=None): yield "trace-id" - def score(self, _tid, *, name, value, data_type): + def score(self, _tid, *, name, value, data_type, comment=None): self.scores[name] = value def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): @@ -499,3 +500,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_eval_error_scores.py b/packages/gooddata-eval/tests/test_eval_error_scores.py new file mode 100644 index 000000000..b5e4b55e5 --- /dev/null +++ b/packages/gooddata-eval/tests/test_eval_error_scores.py @@ -0,0 +1,162 @@ +# (C) 2026 GoodData Corporation +"""Every agentic loop kind writes its Langfuse scores the same way (QA-30213, phase 1). + +A run labelled ERROR gets an extra ``eval_error`` score beside its usual ones; every other run, +a chat API error included, is scored exactly as before and gets none. The visualization kind +has its own tests in test_outcome.py. +""" + +from collections import defaultdict +from collections.abc import Callable, Iterator +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import alert_skill, anomaly_detection, dashboard_skill, kda_skill, what_if +from gooddata_eval.core.agentic._outcome import RunLabels +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.outcome import Outcome + + +@dataclass(frozen=True) +class _Kind: + module: Any + evaluate: Callable[..., Any] + expected_output: dict + simulated_user: str | None + + +_KINDS = { + "alert_skill": _Kind( + alert_skill, + alert_skill.evaluate_agentic_alert_skill, + {"operator": "GREATER_THAN", "threshold": 100}, + "generate_simulated_alert_response", + ), + "kda_skill": _Kind( + kda_skill, + kda_skill.evaluate_agentic_kda_skill, + {"Measure": {"type": "metric", "id": "revenue"}}, + "generate_simulated_kda_response", + ), + "dashboard_skill": _Kind( + dashboard_skill, + dashboard_skill.evaluate_agentic_dashboard_skill, + { + "type": "dashboard", + "saved_dashboard_id": None, + "visualizations": [{"id": None, "title": "Revenue"}], + "date_range": None, + "min_new_visualizations": 1, + }, + None, + ), + "anomaly_detection": _Kind( + anomaly_detection, + anomaly_detection.evaluate_agentic_anomaly_detection, + {"metric": "metric/spend", "granularity": "MONTH"}, + "generate_simulated_anomaly_response", + ), + "what_if": _Kind( + what_if, + what_if.evaluate_agentic_what_if, + {"metric_id": "revenue"}, + "generate_simulated_what_if_response", + ), +} + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + def run_name(self, run_idx: int) -> str: + return f"run{run_idx}" + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(kind: _Kind, labels: RunLabels | None = None) -> _ScoreCtx: + """Run 1's chat call fails after its retries; run 2 answers without doing the task. + + ``labels`` replaces the labels the kind computes, to give run 1 a harness ERROR without + driving each kind's own simulated-user loop.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate({"textResponse": "I cannot help with that."}), + ] + captured: dict[str, Any] = {} + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + captured["write_scores"] = kwargs["write_scores"] + + with ExitStack() as stack: + stack.enter_context(patch.object(kind.module, "ChatClient", return_value=client)) + stack.enter_context(patch.object(kind.module, "submit_trace_scoring", _capture)) + if labels is not None: + stack.enter_context(patch.object(kind.module, "label_runs", return_value=labels)) + if kind.simulated_user is not None: + stack.enter_context(patch.object(kind.module, kind.simulated_user, return_value="please go on")) + try: + kind.evaluate( + "http://host/api/v1/actions/workspaces/ws1/ai", + "tok", + "ws1", + "Do the thing", + kind.expected_output, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + except AssertionError: + pass # the scored run fails; its scores are written before the raise + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +_USUAL_SCORES = {"pass_at_k", "pass_power_k", "gate_passed"} + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_an_errored_run_gets_eval_error_beside_its_usual_scores(name: str) -> None: + labels = RunLabels((Outcome.error("simulated_user_failed"), Outcome.failed("agent_silent"))) + ctx = _scores(_KINDS[name], labels) + + assert ("eval_error", 1, "simulated_user_failed") in ctx.scores[0] + assert set(ctx.names(0)) >= _USUAL_SCORES + assert "eval_error" not in ctx.names(1) + assert set(ctx.names(1)) >= _USUAL_SCORES + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_a_chat_error_run_is_scored_as_before_with_no_marker(name: str) -> None: + ctx = _scores(_KINDS[name]) + + for run_idx in (0, 1): + names = ctx.names(run_idx) + assert "eval_error" not in names + assert set(names) >= _USUAL_SCORES diff --git a/packages/gooddata-eval/tests/test_outcome.py b/packages/gooddata-eval/tests/test_outcome.py new file mode 100644 index 000000000..71b75bd6f --- /dev/null +++ b/packages/gooddata-eval/tests/test_outcome.py @@ -0,0 +1,486 @@ +# (C) 2026 GoodData Corporation +"""PASS / FAIL / ERROR labels per run (QA-30213, phase 1): they explain, they never rescore. + +A run is ERROR only when the harness failed -- its simulated user or its judge. Every other +non-pass, a chat API error and the agent's own timeout included, is FAIL. The labels change no +gate, count or message head: an ERROR run still counts as a run that did not pass. +""" + +from collections import defaultdict +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.agentic._outcome import chat_failure_exit, label_runs, mark_unscored_run +from gooddata_eval.core.agentic.document_skill import _judge_error_of +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.models import AgenticAssertionError, ChatResult, CreatedVisualization, LoopExit +from gooddata_eval.core.outcome import ( + JUDGE_FAILED, + Outcome, + OutcomeStatus, + classify_run, + item_outcome, + shadow_verdict, +) + +PASS = Outcome.passed() +FAIL = Outcome.failed("agent_silent") +SIM = Outcome.error("simulated_user_failed") +JUDGE = Outcome.error(JUDGE_FAILED) + + +# ── the rule: an allowlist of harness causes, everything else is FAIL ───────── + +# Every LoopExit, with the label of a run that ended on it without passing. Exhaustive on +# purpose: a new member must be classified here before the suite passes. +_EXPECTED = { + LoopExit.SUCCESS: OutcomeStatus.FAIL, + LoopExit.AGENT_SILENT: OutcomeStatus.FAIL, + LoopExit.BUDGET_EXHAUSTED: OutcomeStatus.FAIL, + LoopExit.STOPPED_EARLY: OutcomeStatus.FAIL, + LoopExit.SIMULATED_USER_FAILED: OutcomeStatus.ERROR, + LoopExit.CHAT_ERROR: OutcomeStatus.FAIL, + LoopExit.AGENT_TIMEOUT: OutcomeStatus.FAIL, + LoopExit.TURN_INCOMPLETE: OutcomeStatus.FAIL, + LoopExit.NOT_RUN: OutcomeStatus.FAIL, +} + + +def test_every_loop_exit_is_classified() -> None: + assert set(_EXPECTED) == set(LoopExit) + + +@pytest.mark.parametrize("loop_exit", list(LoopExit)) +def test_each_loop_exit_has_its_label_and_names_itself_as_the_reason(loop_exit: LoopExit) -> None: + outcome = classify_run(loop_exit, passed=False) + assert outcome.status is _EXPECTED[loop_exit] + assert outcome.reason == loop_exit.value + assert classify_run(loop_exit.value, passed=False) == outcome + + +@pytest.mark.parametrize("loop_exit", list(LoopExit)) +def test_a_run_that_passed_is_pass_whatever_its_exit(loop_exit: LoopExit) -> None: + assert classify_run(loop_exit, passed=True) == PASS + + +def test_a_judge_failure_is_an_error_whatever_the_exit() -> None: + assert classify_run(LoopExit.SUCCESS, passed=False, judge_error="empty body twice") == JUDGE + assert classify_run(None, passed=False, judge_error="unparseable JSON") == JUDGE + + +def test_an_unknown_cause_is_a_failure_never_an_error() -> None: + assert classify_run("some_new_exit", passed=False) == Outcome.failed("some_new_exit") + assert classify_run(None, passed=False) == Outcome.failed(None) + + +def test_an_outcome_record_is_json_ready() -> None: + assert SIM.as_record() == {"status": "error", "reason": "simulated_user_failed"} + assert Outcome.failed("chat_error").as_record() == {"status": "fail", "reason": "chat_error"} + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("503 after retries"), LoopExit.CHAT_ERROR), + (ChatError("SSE stream error"), LoopExit.CHAT_ERROR), + (httpx.ReadError("connection reset"), LoopExit.CHAT_ERROR), + ], +) +def test_a_chat_failure_maps_to_its_own_exit_and_is_a_failure(exc: BaseException, loop_exit: LoopExit) -> None: + assert chat_failure_exit(exc) is loop_exit + assert classify_run(chat_failure_exit(exc), passed=False).status is OutcomeStatus.FAIL + + +# ── the item: ERROR only when no run was scored ────────────────────────────── + + +def test_an_item_is_an_error_only_when_every_run_errored() -> None: + assert item_outcome([SIM, SIM, SIM]) == SIM + assert item_outcome([SIM, PASS, PASS]) is None + assert item_outcome([SIM, FAIL]) is None + assert item_outcome([]) is None + + +def test_an_errored_item_names_its_most_frequent_reason() -> None: + assert item_outcome([SIM, JUDGE, JUDGE]) == JUDGE + + +# ── the shadow verdict: the phase-2 candidate rule, recorded and never used ── + + +@pytest.mark.parametrize( + ("outcomes", "gate", "verdict", "scored"), + [ + ([SIM, PASS, PASS], "power", "pass", 2), + ([SIM, SIM, PASS], "power", "error", 1), # fewer than ceil(3/2) scored runs + ([SIM, SIM, SIM], "power", "error", 0), + ([Outcome.failed("chat_error"), PASS, PASS], "power", "fail", 3), # a chat error is scored + ([SIM, PASS], "power", "pass", 1), # K=2 needs one scored run + ([SIM], "power", "error", 0), + ([SIM, FAIL, PASS], "any", "pass", 2), + ([PASS, PASS, PASS], "power", "pass", 3), + ], +) +def test_the_shadow_verdict(outcomes: list[Outcome], gate: str, verdict: str, scored: int) -> None: + assert shadow_verdict(outcomes, gate) == {"verdict": verdict, "runs_scored": scored, "runs_total": len(outcomes)} + + +class _Run: + def __init__(self, ok: bool, exit_reason: LoopExit | None = None, judge_error: str | None = None) -> None: + self.ok = ok + self.exit_reason = exit_reason + self.judge_error = judge_error + + +def test_label_runs_reads_each_runs_exit_and_judge() -> None: + runs = [ + _Run(False, LoopExit.SIMULATED_USER_FAILED), + _Run(False, LoopExit.SUCCESS, judge_error="no verdict"), + _Run(True, LoopExit.SUCCESS), + _Run(False, LoopExit.CHAT_ERROR), + ] + labels = label_runs(runs, passed=lambda r: r.ok) + assert labels.outcomes == (SIM, JUDGE, PASS, Outcome.failed("chat_error")) + assert labels.errored_run(0) == SIM + assert labels.errored_run(2) is None + assert labels.errored_run(3) is None + assert labels.errored_run(9) is None + assert labels.item_outcome is None + assert labels.shadow("power") == {"verdict": "fail", "runs_scored": 2, "runs_total": 4} + + +def test_a_kind_can_name_its_own_judge_error() -> None: + run = _Run(False, LoopExit.SUCCESS, judge_error="takeaway request failed") + assert label_runs([run], passed=lambda r: r.ok).outcomes == (JUDGE,) + labels = label_runs([run], passed=lambda r: r.ok, judge_error=lambda r: None) + assert labels.outcomes == (Outcome.failed("success"),) + + +class _Evaluation: + def __init__(self, judge_error: str | None, fill_error: str | None) -> None: + self.judge_error = judge_error + self.fill_error = fill_error + + +class _DocRun: + def __init__(self, judge_error: str | None, fill_error: str | None) -> None: + self.evaluation = _Evaluation(judge_error, fill_error) + self.exit_reason = None + + +def test_a_document_run_whose_takeaway_request_failed_is_a_failure_not_a_judge_error() -> None: + request_failed = _DocRun(judge_error=None, fill_error="503 from the takeaway endpoint") + judge_failed = _DocRun(judge_error="empty body twice", fill_error=None) + labels = label_runs([request_failed, judge_failed], passed=lambda r: False, judge_error=_judge_error_of) + assert labels.outcomes == (Outcome.failed(None), JUDGE) + + +def test_an_unscored_run_gets_the_marker_only_when_the_harness_failed() -> None: + ctx = MagicMock() + mark_unscored_run(ctx, 0, "conv-1", JUDGE) + ctx.export_unscored.assert_not_called() + ctx.score.assert_called_once() + assert ctx.score.call_args.kwargs == { + "name": "eval_error", + "value": 1, + "data_type": "NUMERIC", + "comment": "judge_failed", + } + + ctx = MagicMock() + mark_unscored_run(ctx, 1, "conv-2", None) + ctx.export_unscored.assert_called_once_with(1, "conv-2") + ctx.score.assert_not_called() + + +# ── end to end through the visualization kind: labels, never a new verdict ─── + + +def _viz() -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn() -> ChatResult: + return ChatResult.model_validate({"createdVisualizations": {"objects": [_viz()], "reasoning": ""}}) + + +def _question_turn() -> ChatResult: + return ChatResult.model_validate({"textResponse": "Which metric?"}) + + +def _evaluate(turns: list[Any], captured: dict[str, Any] | None = None) -> object: + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, len(turns) + 1)] + client.send_message.side_effect = turns + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + if captured is not None: + captured["write_scores"] = kwargs["write_scores"] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + patch.object(agentic_visualization, "generate_simulated_response", side_effect=RuntimeError("openai 401")), + patch.object(agentic_visualization, "submit_trace_scoring", _capture), + ): + return evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=len(turns), + gate="power", + langfuse=MagicMock() if captured is not None else None, + dataset_item_id="item-1" if captured is not None else None, + ) + + +def _silent_turn() -> ChatResult: + return ChatResult.model_validate({}) + + +def test_a_simulated_user_failure_keeps_the_verdict_and_is_labelled() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _viz_turn(), _viz_turn()]) + + exc = exc_info.value + message = str(exc) + lines = message.splitlines() + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + # The gate counted the errored run as a run that did not pass: pass^3, 2/3, as before. + assert lines[2].startswith(gate_failure_note("power", 2, 3)) + assert "run 1/3 ERROR (simulated_user_failed)" in lines[2] + assert (exc.runs_passed, exc.runs_effective) == (2, 3) + (record,) = exc.failed_runs + assert record["exit_reason"] == "simulated_user_failed" + assert record["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + # Some runs were scored, so the item is not a harness error. + assert not hasattr(exc, "outcome") + assert exc.shadow == {"verdict": "pass", "runs_scored": 2, "runs_total": 3} + + +def test_an_item_whose_every_run_errored_is_labelled_and_still_fails() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _question_turn(), _question_turn()]) + + exc = exc_info.value + assert isinstance(exc, AgenticAssertionError) + assert exc.outcome == {"status": "error", "reason": "simulated_user_failed"} + assert exc.shadow == {"verdict": "error", "runs_scored": 0, "runs_total": 3} + gate_line = str(exc).splitlines()[2] + assert gate_line.startswith( + f"Harness error, no run was scored (simulated_user_failed). {gate_failure_note('power', 0, 3)}" + ) + assert exc.runs_passed == 0 + assert len(exc.failed_runs) == 3 + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + (ChatError("SSE stream error"), "chat_error"), + ], +) +def test_a_chat_failure_is_a_plain_failure(exc: BaseException, exit_value: str) -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([exc, _viz_turn(), _viz_turn()]) + + raised = exc_info.value + assert (raised.runs_passed, raised.runs_effective) == (2, 3) + (record,) = raised.failed_runs + assert record["exit_reason"] == exit_value + assert record["outcome"] == {"status": "fail", "reason": exit_value} + assert f"run 1/3 FAIL (loop exit: {exit_value})" in str(raised) + assert "ERROR" not in str(raised) + assert not hasattr(raised, "outcome") + assert raised.shadow["verdict"] == "fail" + + +def test_only_timeouts_is_a_failure_not_a_harness_error() -> None: + timeout = TurnTimeoutError("agent exceeded the 60s turn budget") + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([timeout, timeout, timeout]) + + assert not hasattr(exc_info.value, "outcome") + assert str(exc_info.value).splitlines()[2].startswith(gate_failure_note("power", 0, 3)) + + +# ── Langfuse: an ERROR run gets an extra marker, every usual score stays ────── + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(turns: list[Any]) -> _ScoreCtx: + captured: dict[str, Any] = {} + try: + _evaluate(turns, captured) + except AssertionError: + pass + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +_USUAL_SCORES = {"gate_passed", "pass_at_k", "pass_power_k", "quality_score"} + + +def test_an_errored_run_gets_eval_error_beside_its_usual_scores() -> None: + ctx = _scores([_question_turn(), _viz_turn(), _viz_turn()]) + + assert ("eval_error", 1, "simulated_user_failed") in ctx.scores[0] + assert set(ctx.names(0)) >= _USUAL_SCORES + # The verdict scores are the ones the item was judged by: unchanged by the label. + assert dict((n, v) for n, v, _ in ctx.scores[0])["gate_passed"] is False + for run_idx in (1, 2): + assert "eval_error" not in ctx.names(run_idx) + assert set(ctx.names(run_idx)) >= _USUAL_SCORES + + +def test_a_chat_error_run_gets_no_marker() -> None: + ctx = _scores([ChatError("SSE stream error"), _viz_turn(), _viz_turn()]) + + for run_idx in range(3): + assert "eval_error" not in ctx.names(run_idx) + assert set(ctx.names(run_idx)) >= _USUAL_SCORES + + +# ── the message is the master text plus the always-on failing runs ─────────── + +# The message of [simulated user failed, agent silent, pass] under pass^3, as the package +# produced it before labels existed. Only the "Failing runs" part of the gate line is new. +_MASTER_MESSAGE = ( + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + "Agentic Visualization Assertion Failed! (Critical Mode)\n" + "Gate pass^3 failed: 1/3 runs passed — unstable, not a clean failure.\n" + "------------------------------------------\n" + "Question:\n" + "Show revenue by quarter\n" + "------------------------------------------\n" + "Expected Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Actual Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Strict Check Summary:\n" + " Visualization Created : True\n" + " Cross-Ref Valid : True\n" + " Metrics Correct : True\n" + " expected : ['metric/revenue']\n" + " actual : ['metric/revenue']\n" + " Dimensions Correct : True\n" + " expected : ['label/date.quarter']\n" + " actual : ['label/date.quarter']\n" + " Filters Correct : True\n" + " date : True\n" + " ranking : True\n" + " attribute : True\n" + " Viz Type Hard : True\n" + " Executed : False\n" + " Stated Value Matches : None (reported only)\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" +) + + +def test_the_agentic_message_is_the_master_text_plus_the_failing_runs() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _silent_turn(), _viz_turn()]) + + message = str(exc_info.value) + gate_line = message.splitlines()[2] + head, failing = gate_line.split(" Failing runs: ", 1) + assert failing.startswith("run 1/3 ERROR (simulated_user_failed); ") + assert "; run 2/3 FAIL (loop exit: agent_silent): " in failing + assert message.replace(gate_line, head, 1) == _MASTER_MESSAGE