diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index 183cb387c..4b33b57a2 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -34,6 +34,7 @@ experiment. The newest and most actively developed package in the repo. | `core/reporting/` | console and JSON output rendering | | `core/scoring.py`, `core/runner.py` | scoring and orchestration | | `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends | +| `core/outcome.py` | `Outcome` (PASS / FAIL / ERROR): the one fail-closed rule that labels a failing run ERROR when the harness failed (simulated user, judge), plus the item label and the recorded-only shadow verdict. Labels decide no verdict; the agentic kinds reach it through `core/agentic/_outcome.py`'s `label_runs` | **Depends on**: `gooddata-sdk`, `httpx`, `pydantic`, `orjson`, `rich`. The LLM-judge evaluator is an optional extra (`llm-judge`, pulling `openai>=1.45,<2.0`); every `openai` diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index da4619008..e596c6128 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -404,6 +404,16 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None: item_report.runs_ungraded = unscored +def _apply_labels(item_report: ItemReport, source: Any) -> None: + """Copy the item's harness label and shadow verdict off an outcome or a failure, if set.""" + harness_error = getattr(source, "harness_error", None) + if isinstance(harness_error, dict): + item_report.harness_error = harness_error + shadow = getattr(source, "shadow", None) + if isinstance(shadow, dict): + item_report.shadow = shadow + + def _apply_failed_runs(item_report: ItemReport, source: Any) -> None: """Copy the per-run failure records off an outcome or a failure, if the kind built any. @@ -555,6 +565,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: _apply_best_run_latency(item_report, outcome) _apply_run_counts(item_report, outcome) _apply_failed_runs(item_report, outcome) + _apply_labels(item_report, outcome) _apply_run_latencies(item_report, outcome) except AssertionError as exc: item_report.gate_passed = False if gated else None @@ -567,6 +578,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: _apply_best_run_latency(item_report, exc) _apply_run_counts(item_report, exc) _apply_failed_runs(item_report, exc) + _apply_labels(item_report, exc) _apply_run_latencies(item_report, exc) # Read off the counts, not off the gate: pass^K fails items where runs did pass, # and reporting those as pass_at_k False would contradict the Langfuse score of @@ -588,6 +600,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: _apply_timings(item_report, getattr(exc, "timings", None)) _apply_run_counts(item_report, exc) _apply_failed_runs(item_report, exc) + _apply_labels(item_report, exc) _apply_run_latencies(item_report, exc) # Every run ungraded, by definition of this error -- said explicitly rather than # left to _apply_run_counts, whose source is detail["unscored_runs"] and which a diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py index 694928b9b..08bd4f899 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failed_runs.py @@ -25,17 +25,28 @@ from collections.abc import Callable, Sequence from typing import Any +from gooddata_eval.core.agentic._failure_message import cap_failed_checks +from gooddata_eval.core.agentic._outcome import RunLabels, run_exit_reason, run_outcome from gooddata_eval.core.models import build_tool_calls +from gooddata_eval.core.outcome import OutcomeStatus def build_failed_runs( run_results: Sequence[Any], *, - passed: Callable[[Any], bool], detail: Callable[[Any], dict], + labels: RunLabels | None = None, + passed: Callable[[Any], bool] | None = None, + failed_checks: Callable[[Any], list[dict]] | None = None, ) -> list[dict]: """One record per run that did not pass, in the order the runs happened. + A kind that labels its runs passes its ``labels`` (from ``label_runs``, in run order): + they say which runs did not pass and are the ``outcome`` of each record, so the records, + the failure message and the Langfuse ``eval_error`` marker read one set of labels. A kind + that does not label its runs passes ``passed`` instead, and each record's ``outcome`` is + worked out here from the run alone. Exactly one of the two is given. + ``passed`` and ``detail`` are supplied by the caller because neither is uniform across kinds: guardrail reads ``run.passed`` while visualization reads ``run.eval_result.strict_pass``, and each kind's diagnostic dict is its own. Everything @@ -45,28 +56,51 @@ def build_failed_runs( ``ToolCallEvent``s, so a run that has the list must hold real events in it. A run the judge could not grade is recorded like any other non-passing run, with its - ``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the - verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``. + ``error`` set to the run's ``judge_error``. That matches ``core/runner.py``, which also + records ungraded runs; the verdict-level accounting of ungraded runs stays with the + caller's ``unscored_runs``. + + ``exit_reason`` is the run's ``LoopExit`` value, or null for a kind whose loop records + none, and ``outcome`` its ``core.outcome.Outcome`` record: ERROR when the harness failed + (its simulated user, or its judge when the judge's verdict was the one still open), FAIL + otherwise. ``error`` and ``outcome`` can differ: a run that failed a fixed check and whose + judge also broke has ``error`` set and is a FAIL. The label decides nothing; the run counts + as not passed either way. + + ``failed_checks`` is written only when the caller supplies it: a missing key means the + kind does not name its failing checks, while an empty list would claim the run failed + none. Its expected/actual values are capped in size; the run's ``detail`` keeps them whole. """ + if (labels is None) == (passed is None): + raise ValueError("build_failed_runs takes exactly one of labels and passed") + if labels is not None and len(labels.outcomes) != len(run_results): + raise ValueError(f"{len(labels.outcomes)} labels for {len(run_results)} runs") records: list[dict] = [] for run_index, run in enumerate(run_results, start=1): - if passed(run): + if labels is not None: + outcome = labels.outcomes[run_index - 1] + else: + outcome = run_outcome(run, passed=passed is not None and passed(run)) + if outcome.status is OutcomeStatus.PASS: continue tool_calls = list(getattr(run, "tool_call_events", None) or []) reasoning_steps = list(getattr(run, "reasoning_steps", None) or []) - records.append( - { - "run_index": run_index, - "passed": False, - "error": getattr(run, "judge_error", None), - "detail": detail(run), - "conversation_id": getattr(run, "conversation_id", None), - "response_id": getattr(run, "response_id", None), - "reasoning_step_count": len(reasoning_steps), - "reasoning_steps": reasoning_steps, - "tool_call_count": len(tool_calls), - "tool_names": [getattr(e, "function_name", None) for e in tool_calls], - "tool_calls": build_tool_calls(tool_calls), - } - ) + record: dict[str, Any] = { + "run_index": run_index, + "passed": False, + "error": getattr(run, "judge_error", None), + "detail": detail(run), + "conversation_id": getattr(run, "conversation_id", None), + "response_id": getattr(run, "response_id", None), + "reasoning_step_count": len(reasoning_steps), + "reasoning_steps": reasoning_steps, + "tool_call_count": len(tool_calls), + "tool_names": [getattr(e, "function_name", None) for e in tool_calls], + "tool_calls": build_tool_calls(tool_calls), + "exit_reason": run_exit_reason(run), + "outcome": outcome.as_record(), + } + if failed_checks is not None: + record["failed_checks"] = cap_failed_checks(failed_checks(run)) + records.append(record) return records diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py new file mode 100644 index 000000000..d05a93087 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_failure_message.py @@ -0,0 +1,198 @@ +# (C) 2026 GoodData Corporation +"""The head of an agentic kind's failure message: the gate result, then every failing run. + +Each kind's assertion message describes its BEST run, which on a partial gate failure (some +of the K runs passed) is a passing run: every check reads True and the reason the item +failed is nowhere in the message. This module renders the part that is missing -- for each +failing run its index, why its loop stopped and the checks it failed, with expected/actual +where the kind knows them -- from the same ``failed_runs`` records the item reports, so the +message and the triage data cannot disagree about which runs failed or why. + +One renderer for every kind that labels its runs, on purpose: the readers of these messages +(people and the triage tooling) look for one shape, whatever the capability under test. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from gooddata_eval.core.outcome import UNKNOWN_REASON, OutcomeStatus + +__all__ = [ + "FailedCheck", + "cap_failed_checks", + "failed_check_records", + "failed_checks_from", + "failing_runs_note", + "failure_head", + "judge_failed_checks", +] + +# A value longer than this is cut: the message is one line per run, and the whole value is in +# the run's detail anyway. +_MAX_VALUE_CHARS = 160 + +# The failing runs together are cut past this, at a run boundary: K runs of a kind with a dozen +# checks would otherwise put tens of thousands of characters into one assertion message. Every +# run is in ``failed_runs`` whatever the message shows. +_MAX_FAILING_RUNS_CHARS = 2000 + + +class _Unset(Enum): + UNSET = "unset" + + +_UNSET = _Unset.UNSET + + +@dataclass(frozen=True) +class FailedCheck: + """One check a run failed, with the expected/actual pair when the kind knows it. + + ``part`` narrows the check when one name covers several parts, e.g. the ``date`` part of + ``filters_correct``. + """ + + name: str + expected: Any = _UNSET + actual: Any = _UNSET + part: str | None = None + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into a failing run's ``failed_checks``.""" + record: dict[str, Any] = {"name": self.name} + if self.part is not None: + record["part"] = self.part + if self.expected is not _UNSET: + record["expected"] = self.expected + if self.actual is not _UNSET: + record["actual"] = self.actual + return record + + +def failed_check_records(checks: Iterable[FailedCheck]) -> list[dict[str, Any]]: + """``checks`` as the records ``failed_runs[].failed_checks`` holds.""" + return [check.as_record() for check in checks] + + +def failed_checks_from(checks: Mapping[str, bool], *, prerequisite: str | None = None) -> list[dict[str, Any]]: + """The records of the checks in ``checks`` that read False, by name, in their order. + + ``prerequisite`` names the check every other one depends on, e.g. ``alert_created``. When it + failed, the others read False only because there was nothing to check, so it alone is named. + """ + if prerequisite is not None and checks.get(prerequisite) is False: + return failed_check_records([FailedCheck(prerequisite)]) + return failed_check_records(FailedCheck(name) for name, ok in checks.items() if not ok) + + +def judge_failed_checks(name: str, *, passed: bool, judge_error: str | None, reasoning: Any) -> list[dict[str, Any]]: + """The judge's verdict ``name`` on a run it failed, with its reasoning as the actual value. + + Empty when the judge passed the run, or could not grade it: a run with no verdict failed no + judge check. + """ + if judge_error is not None or passed: + return [] + return failed_check_records([FailedCheck(name, part="judge", actual=reasoning)]) + + +def _text(value: Any) -> str: + text = value if isinstance(value, str) else json.dumps(value, default=str, ensure_ascii=False, sort_keys=True) + return " ".join(text.split()) + + +def _value(value: Any) -> str: + if value is None or value == [] or value == {} or value == "": + return "none" + text = _text(value) + return text if len(text) <= _MAX_VALUE_CHARS else text[: _MAX_VALUE_CHARS - 3] + "..." + + +def cap_failed_checks(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """``records`` with every expected/actual value cut to what the message shows of it. + + A check's values can be whole filter or bucket structures, repeated for every failing run + of every item; the record keeps the same ``_MAX_VALUE_CHARS`` the message does. A value + that fits stays as it is, JSON and all; a longer one becomes its cut text. The run's + ``detail`` holds the full values. + """ + capped: list[dict[str, Any]] = [] + for record in records: + entry = dict(record) + for key in ("expected", "actual"): + if key in entry and len(_text(entry[key])) > _MAX_VALUE_CHARS: + entry[key] = _value(entry[key]) + capped.append(entry) + return capped + + +def _render_check(record: Mapping[str, Any]) -> str: + name = str(record.get("name", "?")) + parts: list[str] = [] + if "expected" in record: + parts.append(f"expected {_value(record['expected'])}") + if "actual" in record: + parts.append(f"actual {_value(record['actual'])}") + part = record.get("part") + if not parts and part is None: + return name + inner = ", ".join(parts) + if part is not None: + inner = f"{part}: {inner}" if inner else str(part) + return f"{name}[{inner}]" + + +def _render_run(record: Mapping[str, Any], runs_total: int) -> str: + outcome = record.get("outcome") + head = f"run {record.get('run_index', '?')}/{runs_total}" + exit_reason = record.get("exit_reason") + errored = isinstance(outcome, Mapping) and outcome.get("status") == OutcomeStatus.ERROR.value + if errored: + # A harness error: the reason (simulated_user_failed, judge_failed) is the diagnosis. + # Its checks are still listed: the simulated user is also asked after an answer that + # was already final, and what that answer missed is the only way to tell. + head += f" ERROR ({outcome.get('reason') or UNKNOWN_REASON})" + else: + head += " FAIL" + (f" (loop exit: {exit_reason})" if exit_reason else "") + checks = record.get("failed_checks") + if checks: + return f"{head}: {', '.join(_render_check(c) for c in checks)}" + if checks is not None and not errored: + return f"{head}: no failing check recorded" + return head + + +def failing_runs_note(failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """Every failing run, joined with ``; ``. Empty when no run failed. + + Past ``_MAX_FAILING_RUNS_CHARS`` the runs left are counted instead of rendered. The first + run is always shown, cut to that length if it alone is longer. + """ + shown: list[str] = [] + length = 0 + for index, record in enumerate(failed_runs): + text = _render_run(record, runs_total) + if shown and length + len("; ") + len(text) > _MAX_FAILING_RUNS_CHARS: + more = len(failed_runs) - index + shown.append(f"{more} more failing run(s) in failed_runs") + break + if not shown and len(text) > _MAX_FAILING_RUNS_CHARS: + text = text[: _MAX_FAILING_RUNS_CHARS - 3] + "..." + shown.append(text) + length += len(text) + (len("; ") if len(shown) > 1 else 0) + return "; ".join(shown) + + +def failure_head(gate_note: str, failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: + """``gate_note`` followed by the failing runs, ready to start a kind's assertion message. + + Returns ``gate_note`` unchanged when there is no failing run to list, which is the whole + head of a kind that builds no ``failed_runs``. + """ + note = failing_runs_note(failed_runs, runs_total) + return f"{gate_note} Failing runs: {note}." if note else gate_note diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py index 63980bb07..ec9ace87f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py @@ -29,20 +29,27 @@ def gate_label(gate: str | None, k: int) -> str: return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}" -def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0) -> str: +def gate_failure_note( + gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0, *, runs_errored: int = 0 +) -> str: """Gate and how many runs met it, for the assertion message. Needed because the message body describes the BEST run, which under pass^K can be a run that passed — so the reported detail on its own looks like a pass. An ungraded run counts in runs_total but can never count in runs_passed, so the remainder - is not evidence of instability — saying so would blame the agent for a judge outage. + is not evidence of instability — saying so would blame the agent for a judge outage. A run + the harness failed (``runs_errored``, labelled ERROR in ``core/outcome.py``) is no evidence + of instability either, so it is named rather than called one. A pass^K item with a passing + run and a run the agent itself failed is unstable all the same, whatever else errored. """ label = gate_label(gate, runs_total) note = f"Gate {label} failed: {runs_passed}/{runs_total} runs passed" if runs_ungraded: return f"{note}, {runs_ungraded} ungraded." - if normalize_gate(gate) == "power" and runs_passed: + if runs_errored: + note = f"{note}, {runs_errored} harness error(s)" + if normalize_gate(gate) == "power" and runs_passed and runs_passed + runs_errored < runs_total: return f"{note} — unstable, not a clean failure." return f"{note}." diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py index 576aee838..11df14b36 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_outcome.py @@ -1,5 +1,5 @@ # (C) 2026 GoodData Corporation -"""The common tail of every ``evaluate_agentic_*`` function. +"""The common tail of every ``evaluate_agentic_*`` function, and the run labels it carries. Before this module, each of the agentic evaluators hand-wrote the same block: copy ``reasoning_steps``/``conversation_id``/``response_id``/``detail``/``runs_passed``/ @@ -15,22 +15,200 @@ ``timeline_detail`` builds both ``latency_breakdown`` and ``tool_calls`` from the same events so they stay index-aligned, but roughly half the evaluators called ``build_latency_breakdown`` directly and silently never got a ``tool_calls`` key. + +The run labels (``core/outcome.py``) are applied here for the same reason: ``label_runs`` +labels an item's runs once, and the failure message, the Langfuse marker and the fields on +the raised or returned result all read that one ``RunLabels``. ``chat_failure_exit`` names +the loop exit of a run whose chat call raised, for every kind alike. """ from __future__ import annotations +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum from typing import Any, NoReturn +from gooddata_eval.core.agentic._failure_message import failure_head +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, + LoopExit, ReasoningStepEvent, ToolCallEvent, timeline_detail, ) +from gooddata_eval.core.outcome import ( + EVAL_ERROR_SCORE, + JUDGE_FAILED, + Outcome, + classify_run, + item_outcome, + shadow_verdict, +) from gooddata_eval.core.timing import PhaseTimings -__all__ = ["agentic_detail", "agentic_success", "raise_agentic_failure"] +__all__ = [ + "CHAT_FAILURE_EXITS", + "EVAL_ERROR_SCORE", + "RunLabels", + "agentic_detail", + "agentic_success", + "attach_labels", + "chat_failure_exit", + "failure_message_head", + "label_runs", + "log_eval_error_score", + "mark_unscored_run", + "raise_agentic_failure", + "run_exit_reason", + "run_outcome", +] + + +CHAT_FAILURE_EXITS = frozenset({LoopExit.CHAT_ERROR, LoopExit.AGENT_TIMEOUT, LoopExit.TURN_INCOMPLETE}) +"""Every exit ``chat_failure_exit`` can return: the loop stopped because a chat call raised.""" + + +def chat_failure_exit(exc: BaseException) -> LoopExit: + """The ``LoopExit`` for a run whose chat call raised ``exc``. + + ``TurnTimeoutError`` and ``TurnIncompleteError`` subclass ``ChatError`` but are the + agent's own failures -- too slow, or no final answer -- so they get exits of their own + instead of reading as a chat API fault. Anything else a kind catches there (a gen-ai + error event, a broken stream, a 5xx after its retries) is ``CHAT_ERROR``. All three are + FAIL in ``core/outcome.py``: a customer hits each of them too. + """ + if isinstance(exc, TurnTimeoutError): + return LoopExit.AGENT_TIMEOUT + if isinstance(exc, TurnIncompleteError): + return LoopExit.TURN_INCOMPLETE + return LoopExit.CHAT_ERROR + + +def run_exit_reason(run: Any) -> str | None: + """The run's ``exit_reason`` as a string, or None for a kind whose run records none.""" + value = getattr(run, "exit_reason", None) + if isinstance(value, Enum): + return str(value.value) + return value if isinstance(value, str) else None + + +JudgeErrorOf = Callable[[Any], str | None] +"""How a kind reads the judge failure that left a run without a verdict, or None. + +The default reads ``run.judge_error``, right for a kind whose judge is its only check. A kind +with other checks returns the judge's error only when the judge's verdict was the one still +open, and leaves out a failure its ``judge_error`` reports from the product path.""" + + +def _default_judge_error(run: Any) -> str | None: + value = getattr(run, "judge_error", None) + return value if isinstance(value, str) else None + + +def run_outcome(run: Any, *, passed: bool, judge_error: JudgeErrorOf | None = None) -> Outcome: + """One run's label: ``classify_run`` over the run's exit and its judge failure, if any.""" + error = (judge_error or _default_judge_error)(run) + return classify_run(run_exit_reason(run), passed=passed, judge_error=error if isinstance(error, str) else None) + + +@dataclass(frozen=True) +class RunLabels: + """The label of every run of an item, in run order. + + The labels decide nothing: the kind gates on its own pass_at_k/pass_power_k, and an + ERROR run counts as a run that did not pass. They only say why a run did not pass, and + whether the whole item is a harness error (``item_outcome``). + """ + + outcomes: tuple[Outcome, ...] + + def error_label(self, run_index: int) -> Outcome | None: + """The ERROR label of run ``run_index`` (0-based), or None when that run is not one.""" + outcome = self.outcomes[run_index] if 0 <= run_index < len(self.outcomes) else None + return outcome if outcome is not None and outcome.is_error else None + + @property + def item_outcome(self) -> Outcome | None: + """ERROR when every run errored, else None (see ``core.outcome.item_outcome``).""" + return item_outcome(self.outcomes) + + @property + def harness_error_record(self) -> dict[str, Any] | None: + """``item_outcome`` as the ``harness_error`` record a failed item carries, or None.""" + outcome = self.item_outcome + return outcome.as_record() if outcome is not None else None + + def shadow(self, gate: str | None) -> dict[str, Any]: + """The phase-2 candidate verdict, recorded on every item, passed or failed, and never used.""" + return shadow_verdict(self.outcomes, gate) + + +def label_runs( + run_results: Sequence[Any], *, passed: Callable[[Any], bool], judge_error: JudgeErrorOf | None = None +) -> RunLabels: + """The ``RunLabels`` of an item's runs; ``passed`` is the kind's own per-run verdict.""" + return RunLabels(tuple(run_outcome(r, passed=passed(r), judge_error=judge_error) for r in run_results)) + + +def attach_labels(exc: AgenticAssertionError | JudgeResponseError, labels: RunLabels, gate: str | None) -> None: + """Set ``harness_error`` (only when every run errored) and ``shadow`` on an exception a kind + raises by hand -- the counterpart of the two ``raise_agentic_failure`` arguments.""" + harness_error = labels.harness_error_record + if harness_error is not None: + exc.harness_error = harness_error + exc.shadow = labels.shadow(gate) + + +def failure_message_head( + gate: str | None, labels: RunLabels, runs_passed: int, failed_runs: Sequence[dict], runs_ungraded: int = 0 +) -> str: + """The start of a kind's failure message: the gate result, then every failing run. + + An item whose every run errored says so first. Only when every run's judge failed does it + also say no run was scored: a run whose simulated user failed was still checked, and its + failed checks follow. The gate note itself is the one the item was judged by. + """ + runs_total = len(labels.outcomes) + runs_errored = sum(1 for o in labels.outcomes if o.is_error) + note = gate_failure_note( + gate, runs_passed=runs_passed, runs_total=runs_total, runs_ungraded=runs_ungraded, runs_errored=runs_errored + ) + item = labels.item_outcome + if item is not None: + if all(o.reason == JUDGE_FAILED for o in labels.outcomes): + note = f"Harness error, no run was scored ({JUDGE_FAILED}). {note}" + else: + note = f"Harness error on every run ({item.reason}). {note}" + return failure_head(note, failed_runs, runs_total) + + +def log_eval_error_score(ctx: Any, trace_id: Any, outcome: Outcome) -> None: + """Mark one ERROR run in Langfuse: ``eval_error`` = 1, its comment the reason. + + An extra score only: the run's pass, gate, check and quality scores are written beside + it, unchanged. The marker says why the run failed. + """ + ctx.score(trace_id, name=EVAL_ERROR_SCORE, value=1, data_type="NUMERIC", comment=outcome.reason) + + +def mark_unscored_run(ctx: Any, run_idx: int, conversation_id: str, error: Outcome | None) -> None: + """Langfuse for a run that gets no verdict scores (its judge or its request failed). + + A harness ERROR (``error``, from ``RunLabels.error_label``) is attached to its experiment + item with the ``eval_error`` marker, joined or not, the way every scored run is, so an + errored run is counted wherever the item's runs are. Any other unscored run goes through + ``export_unscored``, which exports the root of a joined run only. + """ + if error is None: + ctx.export_unscored(run_idx, conversation_id) + return + with ctx.observe(ctx.trace(conversation_id), run_idx, conversation_id=conversation_id) as tid: + log_eval_error_score(ctx, tid, error) def agentic_detail( @@ -60,6 +238,8 @@ def raise_agentic_failure( timings: PhaseTimings | None = None, failed_runs: list[dict] | None = None, run_latencies_s: list[float] | None = None, + harness_error: dict[str, Any] | None = None, + shadow: dict[str, Any] | None = None, ) -> NoReturn: """Build ``exception_cls(message)`` with every common field attached, and raise it. @@ -68,6 +248,10 @@ def raise_agentic_failure( "no verdict at all" branch raises a bare ``JudgeResponseError`` instead (not a subclass) does not go through this helper for that branch. + ``harness_error`` is set only when every run errored (``RunLabels.harness_error_record``), and + ``shadow`` is the phase-2 candidate verdict (``RunLabels.shadow``); both are labels a + runner reads, neither changes the failure. + ``timings`` stays optional: only the three kinds that track per-run ``PhaseTimings`` (general_question, metric_skill, dashboard_skill) pass one -- the rest keep their own ``PhaseTimings()`` zero default, same as before this helper existed. ``failed_runs`` @@ -88,6 +272,10 @@ def raise_agentic_failure( exc.failed_runs = failed_runs if run_latencies_s is not None: exc.run_latencies_s = run_latencies_s + if harness_error is not None: + exc.harness_error = harness_error + if shadow is not None: + exc.shadow = shadow raise exc @@ -103,8 +291,13 @@ def agentic_success( timings: PhaseTimings | None = None, failed_runs: list[dict] | None = None, run_latencies_s: list[float] | None = None, + shadow: dict[str, Any] | None = None, ) -> AgenticEvalOutcome: - """The success-path mirror of ``raise_agentic_failure`` -- same fields, same shape.""" + """The success-path mirror of ``raise_agentic_failure`` -- same fields, same shape. + + No ``harness_error``: an item that passed had a run that passed, so it is never one. + ``shadow`` is recorded all the same (``RunLabels.shadow``). + """ kwargs: dict[str, Any] = { "reasoning_steps": reasoning_steps, "conversation_id": conversation_id, @@ -120,4 +313,6 @@ def agentic_success( kwargs["failed_runs"] = failed_runs if run_latencies_s is not None: kwargs["run_latencies_s"] = run_latencies_s + if shadow is not None: + kwargs["shadow"] = shadow return AgenticEvalOutcome(**kwargs) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py index 3f9d3028e..a549f6fef 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py @@ -217,9 +217,10 @@ def export_unscored(self, run_idx: int, conversation_id: str) -> None: with self.observe(self.trace(conversation_id), run_idx, conversation_id=conversation_id): pass - def score(self, trace_id: Any, *, name: str, value: Any, data_type: str) -> None: + def score(self, trace_id: Any, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: """Write one score, swallowing Langfuse failures the way ``score_safe`` always has.""" - self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type) + extra = {"comment": comment} if comment is not None else {} + self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type, **extra) def quality(self, trace_id: Any, *, strict_checks: dict, latency_sec: Any, cost_usd: Any) -> None: """Write the derived quality/value scores for one run.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index f1f532af3..ef1021942 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -15,15 +15,23 @@ from gooddata_eval.core.agentic._catalog import AnomalyDetectionGranularity, CatalogMetricAlert from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1021,7 +1029,7 @@ def _run_once(conv_id: str) -> AlertRunResult: partial_alert_id, _, partial_tool_called = _extract_alert_call(partial.tool_call_events or []) if partial_tool_called: alert_id_to_delete = partial_alert_id - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break # `turns_used` counts attempts (set above, so a failed send still shows the # loop reached this iteration); `turns`/`steps` count completed work, so they @@ -1164,6 +1172,14 @@ def _run_detail(run: AlertRunResult) -> dict: } +def _failed_checks(run: AlertRunResult) -> list[dict]: + """The checks this run failed, by name. The arguments the run sent are in its detail. + + Every other check reads False when no alert was created, so then only that one is named. + """ + return failed_checks_from(run.eval.checks(), prerequisite="alert_created") + + def evaluate_agentic_alert_skill( host: str, token: str, @@ -1220,6 +1236,9 @@ def evaluate_agentic_alert_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.eval.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -1231,6 +1250,8 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.eval.checks() with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -1286,12 +1307,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.eval.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) raise_agentic_failure( AlertSkillAssertionError, f"Alert skill assertion failed. {gate_note} strict_pass={ev.strict_pass}. " @@ -1302,6 +1324,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"attributes_correct={ev.attributes_correct}, " f"granularity_correct={ev.granularity_correct}. " f"Actual args: {best.actual_alert_arguments}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -1313,6 +1337,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py index 25f2281a0..27db1b20b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/anomaly_detection.py @@ -33,15 +33,23 @@ from typing import Any from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -259,18 +267,21 @@ class AnomalyEvaluation: asserted: list[str] = field(default_factory=list) disambiguated: bool = False + @property + def strict_checks(self) -> dict[str, bool]: + """Every check strict_pass requires, by name.""" + return { + "triggered": self.triggered, + "executed": self.executed, + "success": self.success, + "turn_completed": self.turn_completed, + "metric_correct": self.metric_correct, + "granularity_correct": self.granularity_correct, + } + @property def strict_pass(self) -> bool: - return all( - [ - self.triggered, - self.executed, - self.success, - self.turn_completed, - self.metric_correct, - self.granularity_correct, - ] - ) + return all(self.strict_checks.values()) @dataclass @@ -421,7 +432,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) viz_args, execute_result = _extract_anomaly_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -535,6 +546,11 @@ def _detail(best: AnomalyRunResult) -> dict[str, Any]: ) +def _failed_checks(run: AnomalyRunResult) -> list[dict]: + """The checks this run failed, by name. What the agent sent is in the run's detail.""" + return failed_checks_from(run.evaluation.strict_checks) + + def evaluate_agentic_anomaly_detection( host: str, token: str, @@ -583,6 +599,9 @@ def evaluate_agentic_anomaly_detection( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -613,6 +632,8 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -654,13 +675,14 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_detail, + failed_checks=_failed_checks, ) runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) message = ( f"Anomaly detection assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " @@ -672,6 +694,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( AnomalyDetectionAssertionError, message, + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -684,6 +708,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 92f106340..5d227c9e8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -56,7 +56,13 @@ summarize_visualizations, ) from gooddata_eval.core.agentic._gate import log_gate_scores -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1038,7 +1044,7 @@ def run_agentic_conversation( conversation_reasoning_step_events.extend(partial.reasoning_step_events or []) reasoning_steps.extend(partial.reasoning_steps or []) response_id = partial.response_id or response_id - turn_exit = LoopExit.CHAT_ERROR + turn_exit = chat_failure_exit(exc) break final_result = chat_result total_steps += chat_result.reasoning_step_count @@ -1255,7 +1261,7 @@ def run_agentic_conversation( # A chat fault used to escape the whole run, so reaching here did mean no # error. Now that it is caught and recorded, this has to read it back -- # otherwise a turn whose chat call failed reports no_error=True. - no_error=turn_exit is not LoopExit.CHAT_ERROR, + no_error=turn_exit not in CHAT_FAILURE_EXITS, activated_skills=declared or [], active_skills=sorted(active_skills), clarification_turns_used=clarification_turns, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py index b7f2739f9..72f7e2d4e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/dashboard_skill.py @@ -13,16 +13,24 @@ from gooddata_eval.core.agentic._clarification import all_questions from gooddata_eval.core.agentic._conversation_context import CONFIRMATION_REPLY, asks_in_text, classify_reply, clip from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._filters import describe_selection, same_unordered_values, selection_of from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1903,7 +1911,8 @@ class DashboardRunResult: run_latency_s: float = 0.0 # Why the loop stopped -- see LoopExit. A dashboard not produced alone cannot tell a # refusal from a run that hit max_iterations while still on track (matches - # kda_skill.py/alert_skill.py). CHAT_ERROR is reachable; SIMULATED_USER_FAILED is not -- + # kda_skill.py/alert_skill.py). The chat failure exits are reachable (see + # chat_failure_exit); SIMULATED_USER_FAILED is not -- # this loop's follow-up is built from the expectation by build_simulated_reply, with no # judge call that could fail. exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED @@ -2302,7 +2311,7 @@ def _execute_single_dashboard_run( all_tool_call_events.extend(partial.tool_call_events or []) all_reasoning_step_events.extend(partial.reasoning_step_events or []) steps += partial.reasoning_step_count - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -2454,10 +2463,17 @@ def _run_detail(run: DashboardRunResult) -> dict[str, Any]: **run.evaluation.diagnostics, "failures": run.evaluation.failures, "notes": run.evaluation.notes, + # Why the loop stopped -- see LoopExit. + "exit_reason": run.exit_reason.value, "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), } +def _failed_checks(run: DashboardRunResult) -> list[dict]: + """The checks this run failed, by name. What each one found is in the run's ``failures``.""" + return failed_checks_from(run.evaluation.strict_checks) + + def run_agentic_dashboard_skill( host: str, token: str, @@ -2580,6 +2596,9 @@ def evaluate_agentic_dashboard_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -2591,6 +2610,8 @@ def _write_scores(ctx: RunTraceContext) -> None: pt = ctx.trace(run.conversation_id) strict_checks = run.evaluation.strict_checks with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") for name, value in run.evaluation.diagnostics.items(): @@ -2640,12 +2661,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) # Two failures wear the same missing skill, and the advice differs. If the other # dashboard skill was activated, the flag that registers both is plainly on and the # model simply routed the wrong way. If neither was, the flag is off or the agent has @@ -2669,6 +2691,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"Checks: {best.evaluation.strict_checks}. " f"Failures: {'; '.join(best.evaluation.failures) or 'none reported'}." + (f" Notes (not scored): {notes}." if notes else ""), + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -2681,6 +2705,7 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s=[r.run_latency_s for r in summary.run_results], ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index 1b69e2175..7faaa63a7 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -20,11 +20,11 @@ from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._outcome import attach_labels, failure_message_head, label_runs, mark_unscored_run from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -1716,6 +1716,16 @@ class DocumentSkillAssertionError(AgenticAssertionError): """Raised when a document-skill evaluation fails.""" +def _judge_error_of(run: DocumentRunResult) -> str | None: + """The judge's own failure, for the harness label, when it is what left the run without a verdict. + + A run that already failed a fixed check is a FAIL whatever the judge would have said, so its + judge error does not make it a harness error. ``run.judge_error`` also reports a failed + takeaway request (``fill_error``), which is on the product path and stays a FAIL. + """ + return run.evaluation.judge_error if run.evaluation.ungraded else None + + def evaluate_agentic_document_skill( host: str, token: str, @@ -1795,6 +1805,9 @@ def evaluate_agentic_document_skill( document_format=document_format, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, judge_error=_judge_error_of) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -1806,7 +1819,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.ungraded: # No verdict decided this run: its other scores would publish a pass the gate # never counted. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.error_label(run_idx)) continue pt = ctx.trace(run.conversation_id) strict_checks = run.evaluation.strict_checks @@ -1866,6 +1879,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) exc_judge.timings = item_timings exc_judge.run_latencies_s = [r.run_latency_s for r in summary.run_results] + attach_labels(exc_judge, run_labels, gate) raise exc_judge runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) @@ -1896,10 +1910,14 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: } # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. - failed_runs = build_failed_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass, detail=_run_detail) + failed_runs = build_failed_runs( + summary.run_results, + labels=run_labels, + detail=_run_detail, + ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) skill_note = "" if not best.evaluation.wire_names_current: skill_note = " gen-ai still sends the Report names, so it predates the Document rename." @@ -1923,9 +1941,11 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: exc.runs_effective = runs_effective exc.failed_runs = failed_runs exc.run_latencies_s = [r.run_latency_s for r in summary.run_results] + attach_labels(exc, run_labels, gate) raise exc return AgenticEvalOutcome( runs_passed=runs_passed, + shadow=run_labels.shadow(gate), runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 907f0567a..7880dd021 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -7,15 +7,23 @@ from dataclasses import dataclass, field from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import judge_failed_checks from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + attach_labels, + failure_message_head, + label_runs, + mark_unscored_run, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -276,6 +284,13 @@ def _run_detail(run: GeneralQuestionResult) -> dict: } +def _failed_checks(run: GeneralQuestionResult) -> list[dict]: + """The judge's verdict on a run it failed, with its reasoning. A run it could not grade has no verdict.""" + return judge_failed_checks( + "general_question_pass", passed=run.passed, judge_error=run.judge_error, reasoning=run.reasoning + ) + + def evaluate_agentic_general_question( host: str, token: str, @@ -339,6 +354,9 @@ def evaluate_agentic_general_question( document_format=document_format, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.judge_error is None and r.passed) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -350,7 +368,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.judge_error is not None: # No verdict for this run: float(run.passed) would write a 0 the judge # never returned. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.error_label(run_idx)) continue pt = ctx.trace(run.conversation_id) with ctx.observe( @@ -410,12 +428,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # the report has to say so. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), ) - # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as - # non-passing here so it is recorded with its judge_error rather than silently dropped. + # An ungraded run has no verdict, so `r.passed` is not a claim about it: its label is not + # PASS, so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.judge_error is None and r.passed, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not summary.scored_run_results: @@ -431,13 +450,16 @@ def _write_scores(ctx: RunTraceContext) -> None: error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, run_latency_s(best.timings) ) error.timings = item_timings + attach_labels(error, run_labels, gate) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) raise_agentic_failure( GeneralQuestionAssertionError, f"General question assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -450,6 +472,7 @@ def _write_scores(ctx: RunTraceContext) -> None: timings=item_timings, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index f9e281c2d..803ec7f3c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -7,15 +7,23 @@ from dataclasses import dataclass, field from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import judge_failed_checks from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + attach_labels, + failure_message_head, + label_runs, + mark_unscored_run, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -247,6 +255,13 @@ def _run_detail(run: GuardrailResult) -> dict: } +def _failed_checks(run: GuardrailResult) -> list[dict]: + """The judge's verdict on a run it failed, with its reasoning. A run it could not grade has no verdict.""" + return judge_failed_checks( + "guardrail_pass", passed=run.passed, judge_error=run.judge_error, reasoning=run.reasoning + ) + + def evaluate_agentic_guardrail( host: str, token: str, @@ -299,6 +314,9 @@ def evaluate_agentic_guardrail( user_context=user_context, ) + # Before the Langfuse scores, so a run the harness failed gets its marker there too. + run_labels = label_runs(summary.run_results, passed=lambda r: r.judge_error is None and r.passed) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -310,7 +328,7 @@ def _write_scores(ctx: RunTraceContext) -> None: if run.judge_error is not None: # No verdict, so nothing truthful to publish: float(run.passed) would # write a 0 the judge never returned. - ctx.export_unscored(run_idx, run.conversation_id) + mark_unscored_run(ctx, run_idx, run.conversation_id, run_labels.error_label(run_idx)) continue pt = ctx.trace(run.conversation_id) with ctx.observe( @@ -368,12 +386,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # pass@K over fewer runs than --runs asked for is a weaker result. **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), ) - # An ungraded run has no verdict, so `r.passed` is not a claim about it -- treat it as - # non-passing here so it is recorded with its judge_error rather than silently dropped. + # An ungraded run has no verdict, so `r.passed` is not a claim about it: its label is not + # PASS, so it is recorded with its judge_error rather than silently dropped. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.judge_error is None and r.passed, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not summary.scored_run_results: @@ -386,13 +405,16 @@ def _write_scores(ctx: RunTraceContext) -> None: _attach_diagnostics( error, best, detail, failed_runs, runs_passed, runs_effective, run_latencies_s, best.run_latency_s ) + attach_labels(error, run_labels, gate) raise error if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective, len(unscored)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, len(unscored)) raise_agentic_failure( GuardrailAssertionError, f"Guardrail assertion failed. {gate_note} passed={best.passed}. Reasoning: {best.reasoning}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -404,6 +426,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 683f8d42d..e92464bf9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -11,15 +11,23 @@ import httpx from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -176,9 +184,19 @@ class KdaEvaluation: turn_completed: bool disambiguated: bool = False + @property + def strict_checks(self) -> dict[str, bool]: + """Every check strict_pass requires, by name.""" + return { + "triggered": self.triggered, + "executed": self.executed, + "success": self.success, + "turn_completed": self.turn_completed, + } + @property def strict_pass(self) -> bool: - return all([self.triggered, self.executed, self.success, self.turn_completed]) + return all(self.strict_checks.values()) @dataclass @@ -329,7 +347,7 @@ def _accumulate(result: ChatResult) -> None: if create_args is not None: turn_wall_clock_sec = partial.turn_wall_clock_sec turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break turns += 1 steps += chat_result.reasoning_step_count @@ -446,6 +464,11 @@ def _run_detail(run: KdaRunResult) -> dict: } +def _failed_checks(run: KdaRunResult) -> list[dict]: + """The checks this run failed, by name. What the agent sent is in the run's detail.""" + return failed_checks_from(run.evaluation.strict_checks) + + def evaluate_agentic_kda_skill( host: str, token: str, @@ -499,6 +522,9 @@ def evaluate_agentic_kda_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -525,6 +551,8 @@ def _write_scores(ctx: RunTraceContext) -> None: "[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, turn_wall_clock_sec ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN") @@ -585,12 +613,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) message = ( f"KDA skill assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, " @@ -601,6 +630,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( KdaSkillAssertionError, message, + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -612,6 +643,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index f662b205b..17d19182e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -13,15 +13,23 @@ from gooddata_sdk import GoodDataSdk from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -352,7 +360,7 @@ def _execute_single_metric_run( for metric_id in _extract_created_metric_ids(partial.tool_call_events or []): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed @@ -562,6 +570,9 @@ def evaluate_agentic_metric_skill( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.metric_created and r.maql_correct) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -577,6 +588,8 @@ def _write_scores(ctx: RunTraceContext) -> None: conversation_id=run.conversation_id, output={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, ) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") @@ -631,6 +644,25 @@ def _run_detail(run: MetricRunResult) -> dict: **timeline_detail(run.tool_call_events, run.reasoning_step_events), } + def _failed_checks(run: MetricRunResult) -> list[dict]: + """The checks this run failed; the MAQL pair is the whole diagnosis for this kind. + + With no metric, ``maql_correct`` reads False only because there is no MAQL to compare, + so ``metric_created`` alone is named. + """ + checks: list[FailedCheck] = [] + if not run.metric_created: + checks.append(FailedCheck("metric_created", expected=True, actual=False)) + elif not run.maql_correct: + checks.append( + FailedCheck( + "maql_correct", + expected=[c.get("maql", "") for c in expected_outputs_list], + actual=run.actual_maql, + ) + ) + return failed_check_records(checks) + detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, @@ -648,12 +680,13 @@ def _run_detail(run: MetricRunResult) -> dict: # cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.metric_created and r.maql_correct, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) raise_agentic_failure( MetricSkillAssertionError, @@ -661,6 +694,8 @@ def _run_detail(run: MetricRunResult) -> dict: f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " f"Actual MAQL: {best.actual_maql}.", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -673,6 +708,7 @@ def _run_detail(run: MetricRunResult) -> dict: run_latencies_s=run_latencies_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index b06821452..87119eb89 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -12,15 +12,24 @@ from dataclasses import dataclass, field, replace from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -232,21 +241,21 @@ def _execute_single_run( current_result = client.send_message(conversation_id, question) except ChatError as exc: # The opening request, so there is no partial conversation to evaluate: fall through - # to the empty-result path with CHAT_ERROR recorded. Raising here would discard every + # to the empty-result path with its exit recorded. Raising here would discard every # K-run already completed along with any exit_reason. print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}") current_result = getattr(exc, "partial_result", None) or ChatResult() - # The loop that collects events does not run after CHAT_ERROR, so a partial result's + # The loop that collects events does not run after a failed opening request, so a partial result's # chart executions are added here. all_tool_call_events.extend(current_result.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) # Counted here, not at the top of the loop: this request is sent unconditionally, so # with max_iterations=0 the loop body never runs and total_turns would report 0 turns # for a conversation the agent did receive. The loop's own increment is skipped on its # first pass to compensate, keeping total_turns == number of send_message calls. total_turns += 1 - for iteration in range(max_iterations if exit_reason is not LoopExit.CHAT_ERROR else 0): + for iteration in range(max_iterations if exit_reason not in CHAT_FAILURE_EXITS else 0): if iteration: total_turns += 1 total_steps += current_result.reasoning_step_count @@ -297,7 +306,7 @@ def _execute_single_run( # The loop adds a turn's events at its next pass, which a break skips. The # partial result is scored, so its chart executions count too. all_tool_call_events.extend(partial.tool_call_events) - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break skill_activated = _check_visualization_skill_activated(all_tool_call_events) @@ -316,7 +325,7 @@ def _execute_single_run( actual_output, all_tool_call_events, answer_text, - interrupted=exit_reason is LoopExit.CHAT_ERROR, + interrupted=exit_reason in CHAT_FAILURE_EXITS, ) return RunResult( @@ -445,6 +454,81 @@ def _run_detail(run: RunResult) -> dict: } +def _failed_checks(run: RunResult) -> list[dict]: + """The checks this run failed, by their ``detail`` names, with expected/actual where known. + + Follows ``EvaluationResult.strict_pass`` term by term, so the list is empty exactly when + the run passed. + """ + ev = run.eval_result + checks: list[FailedCheck] = [] + if ev.scored_without_chart: + if ev.visualization_created: + actual_type = run.actual_output.type if run.actual_output else None + checks.append(FailedCheck("no_chart", expected="no chart", actual=actual_type)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + return failed_check_records(checks) + if not ev.visualization_created: + # Every chart check reads False without a chart; listing them would name content + # failures for work the agent never did. The answer checks still read the text. + checks.append(FailedCheck("visualization_created", expected=True, actual=False)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + if ev.format_correct is False: + checks.append(FailedCheck("format_correct", actual=ev.format_problems)) + return failed_check_records(checks) + if not ev.cross_ref_valid: + checks.append(FailedCheck("cross_ref_valid", actual=ev.cross_ref_errors)) + if not ev.metrics_correct: + if ev.expected_metric_uris == ev.actual_metric_uris and ev.metric_layout_problems: + # The metric set matched; the order or axis did not, and only these say which. + checks.append(FailedCheck("metrics_correct", part="layout", actual=ev.metric_layout_problems)) + else: + checks.append( + FailedCheck( + "metrics_correct", expected=sorted(ev.expected_metric_uris), actual=sorted(ev.actual_metric_uris) + ) + ) + if not ev.dimensions_correct: + checks.append( + FailedCheck("dimensions_correct", expected=sorted(ev.expected_dim_uris), actual=sorted(ev.actual_dim_uris)) + ) + if not ev.filters_correct: + differing = [ + FailedCheck( + "filters_correct", + part=category, + expected=ev.expected_filters.get(category, []), + actual=ev.actual_filters.get(category, []), + ) + for category in sorted(set(ev.expected_filters) | set(ev.actual_filters)) + if ev.expected_filters.get(category, []) != ev.actual_filters.get(category, []) + ] + checks.extend(differing or [FailedCheck("filters_correct")]) + if not ev.sorts_correct: + checks.append(FailedCheck("sorts_correct", expected=ev.expected_sorts, actual=ev.actual_sorts)) + if not ev.viz_type_hard: + checks.append( + FailedCheck( + "viz_type_hard", + expected=run.best_expected.type if run.best_expected else None, + actual=run.actual_output.type if run.actual_output else None, + ) + ) + if ev.requires_execution and not ev.executed: + checks.append(FailedCheck("executed", expected=True, actual=False)) + if ev.buckets_correct is False: + checks.append(FailedCheck("buckets_correct", expected=ev.expected_buckets, actual=ev.actual_buckets)) + if ev.answer_correct is False: + checks.append(FailedCheck("answer_correct", actual=ev.answer_problems)) + if ev.format_correct is False: + checks.append(FailedCheck("format_correct", actual=ev.format_problems)) + if ev.part_present is False: + checks.append(FailedCheck("part_present", expected=True, actual=False)) + return failed_check_records(checks) + + def _strict_checks(ev: EvaluationResult) -> dict[str, bool]: """The scores one run is gated on, by their Langfuse names. @@ -579,6 +663,9 @@ def evaluate_agentic_visualization( options=options, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.eval_result.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -592,6 +679,8 @@ def _write_scores(ctx: RunTraceContext) -> None: ev = run.eval_result strict_checks = _strict_checks(ev) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id, output=strict_checks) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): # Scored below on every item, gating or not. if score_name != "assertion-vis-executed": @@ -662,12 +751,13 @@ def _write_scores(ctx: RunTraceContext) -> None: # until now only the winning run's survived. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.eval_result.strict_pass, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) n = len(expected_outputs) candidate_note = f" (closest of {n} candidates)" if n > 1 else "" cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else "" @@ -711,6 +801,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f"{_options_lines(ev, best.answer_text)}" f" Stated Value Matches : {ev.stated_value_matches} (reported only)\n" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -722,6 +814,7 @@ def _write_scores(ctx: RunTraceContext) -> None: best_run_latency_s=best.run_latency_s, ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py index de124ceed..c6281b630 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py @@ -27,15 +27,23 @@ from typing import Any from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import failed_checks_from from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -184,20 +192,23 @@ class WhatIfEvaluation: asserted: list[str] = field(default_factory=list) disambiguated: bool = False + @property + def strict_checks(self) -> dict[str, bool]: + """Every check strict_pass requires, by name.""" + return { + "triggered": self.triggered, + "executed": self.executed, + "success": self.success, + "turn_completed": self.turn_completed, + "metric_correct": self.metric_correct, + "maql_correct": self.maql_correct, + "scenario_count_correct": self.scenario_count_correct, + "baseline_correct": self.baseline_correct, + } + @property def strict_pass(self) -> bool: - return all( - [ - self.triggered, - self.executed, - self.success, - self.turn_completed, - self.metric_correct, - self.maql_correct, - self.scenario_count_correct, - self.baseline_correct, - ] - ) + return all(self.strict_checks.values()) @dataclass @@ -378,7 +389,7 @@ def _accumulate(result: ChatResult) -> None: _accumulate(partial) create_args, execute_result = _extract_what_if_calls(all_tool_call_events) turn_completed = False - exit_reason = LoopExit.CHAT_ERROR + exit_reason = chat_failure_exit(exc) break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id @@ -496,6 +507,11 @@ def _detail(best: WhatIfRunResult) -> dict[str, Any]: ) +def _failed_checks(run: WhatIfRunResult) -> list[dict]: + """The checks this run failed, by name. What the agent sent is in the run's detail.""" + return failed_checks_from(run.evaluation.strict_checks) + + def evaluate_agentic_what_if( host: str, token: str, @@ -544,6 +560,9 @@ def evaluate_agentic_what_if( user_context=user_context, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.evaluation.strict_pass) + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -576,6 +595,8 @@ def _write_scores(ctx: RunTraceContext) -> None: } ) with ctx.observe(pt, run_idx, conversation_id=run.conversation_id) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -618,13 +639,14 @@ def _write_scores(ctx: RunTraceContext) -> None: # run is reported with exactly the keys the winning one is. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.evaluation.strict_pass, + labels=run_labels, detail=_detail, + failed_checks=_failed_checks, ) runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) message = ( f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " @@ -637,6 +659,8 @@ def _write_scores(ctx: RunTraceContext) -> None: raise_agentic_failure( WhatIfAssertionError, message, + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -649,6 +673,7 @@ def _write_scores(ctx: RunTraceContext) -> None: ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index ce7ead5c1..78a7388aa 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -340,6 +340,10 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult: return result +def _budget_exceeded(budget: float, scope: str) -> TurnTimeoutError: + return TurnTimeoutError(f"agent exceeded the {budget:.0f}s {scope} budget") + + def _until_deadline( lines: Iterable[str], deadline: float | None, budget: float = 0.0, scope: str = "turn" ) -> Iterable[str]: @@ -351,9 +355,21 @@ def _until_deadline( if deadline is None: yield from lines return - for line in lines: + it = iter(lines) + while True: + try: + line = next(it) + except StopIteration: + return + except httpx.ReadTimeout as exc: + # The read timeout is lowered to the caps, so an agent that goes silent past its + # budget ends here rather than at the check below. Past the deadline it is the same + # timeout; before it, a plain read timeout stays a transport failure. + if time.monotonic() > deadline: + raise _budget_exceeded(budget, scope) from exc + raise if time.monotonic() > deadline: - raise TurnTimeoutError(f"agent exceeded the {budget:.0f}s {scope} budget") + raise _budget_exceeded(budget, scope) yield line @@ -591,16 +607,27 @@ def _do() -> ChatResult: # attempts, but the entire duration of any earlier failed attempt. t0 = time.monotonic() deadline, budget, scope = self._deadline(t0) - with self._client.stream("POST", url, json=body, headers=headers) as resp: - resp.raise_for_status() - try: - result = parse_sse_lines(_until_deadline(resp.iter_lines(), deadline, budget, scope)) - except ChatError as exc: - if exc.partial_result is not None: - exc.partial_result.turn_wall_clock_sec = time.monotonic() - t0 - raise - result.turn_wall_clock_sec = time.monotonic() - t0 - return result + try: + with self._client.stream("POST", url, json=body, headers=headers) as resp: + resp.raise_for_status() + try: + result = parse_sse_lines(_until_deadline(resp.iter_lines(), deadline, budget, scope)) + except ChatError as exc: + if exc.partial_result is not None: + exc.partial_result.turn_wall_clock_sec = time.monotonic() - t0 + raise + result.turn_wall_clock_sec = time.monotonic() - t0 + return result + except httpx.ReadTimeout as exc: + # A read timeout in the stream body is a ChatError by now, so this one came while + # the response headers were awaited. An agent silent past its budget can end here + # too when the headers are held back until the first event. Such a turn has no + # partial result to carry its time, so it is recorded here: it is one of the slowest. + now = time.monotonic() + if deadline is not None and now > deadline: + note_trace_latency(now - t0) + raise _budget_exceeded(budget, scope) from exc + raise try: result = retry_transient(_do, is_retryable=_is_retryable_exc) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py index 6bb4b9e62..2c3230f83 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py @@ -63,6 +63,10 @@ class JudgeResponseError(RuntimeError): failed_runs: list[dict] run_latencies_s: list[float] best_run_latency_s: float | None + # The item's harness label and the phase-2 candidate verdict (``core/outcome.py``), as on + # AgenticAssertionError. + harness_error: dict + shadow: dict def _message_content(response: Any) -> str | None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 094cee08e..391267b78 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -25,9 +25,14 @@ class LoopExit(str, Enum): per-field check as False: specific-sounding content failures for work the agent was never given the chance to do. - This does not change any verdict -- an exhausted run still fails. It makes the three + This does not change any verdict -- an exhausted run still fails. It makes these cases countable, so "is the turn budget too tight" becomes a question the data can answer rather than one that has to be argued. + + Which members label a run ERROR (the harness failed) rather than FAIL is decided in one + place, ``core/outcome.py``. The label decides no verdict. Only SIMULATED_USER_FAILED is an + ERROR: CHAT_ERROR, AGENT_TIMEOUT and TURN_INCOMPLETE are on the product path, and a + customer hits them too. """ SUCCESS = "success" @@ -48,16 +53,27 @@ class LoopExit(str, Enum): SIMULATED_USER_FAILED = "simulated_user_failed" """The harness's own simulated-user model failed to produce a follow-up. A harness-side - fault, not an agent one -- reporting should treat it as an error, not a scored failure.""" + fault, not an agent one, so the run is labelled ERROR (``core/outcome.py``).""" CHAT_ERROR = "chat_error" - """The chat call itself raised mid-conversation. Like SIMULATED_USER_FAILED this is - infrastructure rather than agent capability, but it comes from the GoodData side.""" + """The chat call itself raised mid-conversation and was not one of the agent's own + failures below: a gen-ai error event, a broken stream, or a 5xx still failing after its + retries. It is on the product path -- a customer would see it too -- so the run is a + FAIL, not a harness ERROR.""" + + AGENT_TIMEOUT = "agent_timeout" + """The agent exceeded the per-turn or per-item wall-clock budget (``TurnTimeoutError``). + The agent's own failure to answer in time, so the run is a FAIL.""" + + TURN_INCOMPLETE = "turn_incomplete" + """The agent ended its turn without a final answer, e.g. at its own iteration limit + (``TurnIncompleteError``). The agent's own failure, so the run is a FAIL.""" NOT_RUN = "not_run" """The loop never started -- e.g. an agentic_conversation turn whose $ref to an earlier turn's output could not be resolved, so the turn was skipped before any message was - sent. Distinct from every other member, which all imply at least one turn happened.""" + sent. Distinct from every other member, which all imply at least one turn happened. A + FAIL: its cause is usually the earlier turn's failure, which the item already reports.""" class AacQueryField(BaseModel): @@ -441,6 +457,12 @@ class AgenticAssertionError(AssertionError): failed_runs: list[dict] run_latencies_s: list[float] best_run_latency_s: float | None + harness_error: dict + """``{"status": "error", "reason": ...}``, set only when every run of the item errored + (``core/outcome.py``): the item is a harness error, not an agent failure. A runner labels + it; it stays a failure.""" + shadow: dict + """The phase-2 candidate verdict (``core.outcome.shadow_verdict``). Recorded, never used.""" class AgenticEvalOutcome(BaseModel): @@ -483,6 +505,10 @@ class AgenticEvalOutcome(BaseModel): # not in a loop the runner itself can time. None for a kind that has not yet been # wired to measure it (see AgenticAssertionError.best_run_latency_s, same default). best_run_latency_s: float | None = None + # The phase-2 candidate verdict (``core.outcome.shadow_verdict``), as on + # AgenticAssertionError. Recorded on a passing item too: under pass@K an item can pass with + # most of its runs errored, which the candidate rule would not call a pass. Never used. + shadow: dict | None = None class SummaryInput(BaseModel): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/outcome.py b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py new file mode 100644 index 000000000..0d9d0df95 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/outcome.py @@ -0,0 +1,159 @@ +# (C) 2026 GoodData Corporation +"""One label per run: PASS, FAIL, or ERROR when the test harness, not the agent, failed. + +A run can end in a way that is the harness's fault: the simulated user's own model call +fails, or the judge cannot grade the answer. Scored as usual, such a run reads as an agent +failure. ``classify_run`` names the difference, in one place for every kind. + +The labels decide no verdict. A run labelled ERROR still counts as a run that did not pass, +so a gate, a pass rate or a trend reads the same with or without them. The labels only tell +a reader why a run did not pass. ``shadow_verdict`` records what a candidate rule that leaves +ERROR runs out of the gate would have given, so that rule can be judged on real nights +before anyone adopts it. + +The rule is fail-closed. A run is ERROR only for a cause on ``_ERROR_EXITS`` or a judge +failure that left the run without a verdict, causes a customer would not hit. That is not +proof the agent was blameless: the simulated user is also asked after an answer that was +already final, so an ERROR run still lists the checks it failed. Anything else, including a +chat API error and an unknown exit, is FAIL. +""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from gooddata_eval.core.config import normalize_gate +from gooddata_eval.core.models import LoopExit + +__all__ = [ + "EVAL_ERROR_SCORE", + "JUDGE_FAILED", + "Outcome", + "OutcomeStatus", + "SHADOW_MIN_SCORED_SHARE", + "UNKNOWN_REASON", + "classify_run", + "item_outcome", + "shadow_verdict", +] + + +class OutcomeStatus(str, Enum): + """What one run, or one item, amounts to.""" + + PASS = "pass" + FAIL = "fail" + ERROR = "error" + """The test harness failed, so the run says nothing about the agent.""" + + +@dataclass(frozen=True) +class Outcome: + """A status and a machine-readable reason, e.g. ``simulated_user_failed``.""" + + status: OutcomeStatus + reason: str | None = None + + @classmethod + def passed(cls) -> Outcome: + return cls(OutcomeStatus.PASS) + + @classmethod + def failed(cls, reason: str | None = None) -> Outcome: + return cls(OutcomeStatus.FAIL, reason) + + @classmethod + def error(cls, reason: str) -> Outcome: + return cls(OutcomeStatus.ERROR, reason) + + @property + def is_error(self) -> bool: + return self.status is OutcomeStatus.ERROR + + def as_record(self) -> dict[str, Any]: + """The JSON-ready form written into ``failed_runs[].outcome`` and onto the exception.""" + return {"status": self.status.value, "reason": self.reason} + + +EVAL_ERROR_SCORE = "eval_error" +"""The extra Langfuse score (NUMERIC, 1) an ERROR run gets, its comment the reason. The +run's other scores are written as always.""" + +JUDGE_FAILED = "judge_failed" +"""The reason of a run the judge or scorer could not grade.""" + +UNKNOWN_REASON = "unknown" +"""The reason shown for an ERROR that carries none.""" + +SHADOW_MIN_SCORED_SHARE = 0.5 +"""The share of an item's K runs the candidate rule needs scored, rounded up, before it judges +the item: ceil(K/2). A placeholder until real nights say what it should be.""" + +# The only loop exits that make a run an ERROR. A chat API error stays a FAIL: it is on the +# product path, and a customer would hit it too. +_ERROR_EXITS: frozenset[LoopExit] = frozenset({LoopExit.SIMULATED_USER_FAILED}) + + +def classify_run(exit_reason: LoopExit | str | None, *, passed: bool, judge_error: str | None = None) -> Outcome: + """The label of one run, from whether it passed, why its loop stopped and its judge. + + A run that passed is PASS whatever its exit: its checks held on what the agent produced. + A run whose judge failed is ERROR (``judge_failed``). The caller passes ``judge_error`` only + when the judge's verdict was the one still open: a run that already failed another check is + a FAIL whatever the judge would have said. A run is ERROR too when its exit is on + ``_ERROR_EXITS``. Every other run is FAIL, its reason the exit (or None for a kind whose + loop records none). An exit string that is not a ``LoopExit`` value is kept as the reason + of a FAIL rather than raising: an unknown cause is never an error. + """ + if passed: + return Outcome.passed() + if judge_error is not None: + return Outcome.error(JUDGE_FAILED) + loop_exit: LoopExit | None + if isinstance(exit_reason, LoopExit) or exit_reason is None: + loop_exit = exit_reason + else: + try: + loop_exit = LoopExit(exit_reason) + except ValueError: + return Outcome.failed(exit_reason) + if loop_exit in _ERROR_EXITS: + return Outcome.error(loop_exit.value) + return Outcome.failed(loop_exit.value if loop_exit is not None else None) + + +def item_outcome(outcomes: Sequence[Outcome]) -> Outcome | None: + """ERROR when every run of the item is ERROR, with the most frequent reason; else None. + + Of reasons equally frequent, the alphabetically first is named, so the label is stable. + + None means the item keeps the verdict its gate gave it. An item with some ERROR runs is + not an ERROR: its other runs were scored, and its message names the ERROR runs. + """ + if not outcomes or not all(o.is_error for o in outcomes): + return None + reasons = Counter(o.reason or UNKNOWN_REASON for o in outcomes) + return Outcome.error(max(sorted(reasons), key=lambda reason: reasons[reason])) + + +def shadow_verdict(outcomes: Sequence[Outcome], gate: str | None) -> dict[str, Any]: + """What the candidate phase-2 rule would give this item. Recorded, never used. + + The rule leaves ERROR runs out of K. With fewer scored runs left than + ``SHADOW_MIN_SCORED_SHARE`` of K the item is ``error``; otherwise ``gate`` is applied to + the scored runs. The verdict is an ``OutcomeStatus`` value. + """ + scored = [o for o in outcomes if not o.is_error] + passes = sum(1 for o in scored if o.status is OutcomeStatus.PASS) + if not scored or len(scored) < math.ceil(len(outcomes) * SHADOW_MIN_SCORED_SHARE): + verdict = OutcomeStatus.ERROR + elif normalize_gate(gate) == "power": + verdict = OutcomeStatus.PASS if passes == len(scored) else OutcomeStatus.FAIL + else: + verdict = OutcomeStatus.PASS if passes else OutcomeStatus.FAIL + return {"verdict": verdict.value, "runs_scored": len(scored), "runs_total": len(outcomes)} diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index 71df0abf8..dda6bb711 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -72,6 +72,9 @@ def _build_run_dict(report: EvalReport) -> dict: "runs_passed": item.runs_passed, "runs_ungraded": item.runs_ungraded, "pass_power_k": item.pass_power_k, + # Labels only (core/outcome.py): neither changes gate_passed. + "harness_error": item.harness_error, + "shadow": item.shadow, "best_run_latency_s": ( round(item.best_run_latency_s, 3) if item.best_run_latency_s is not None else None ), diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index 2d95d529a..c93488faa 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -98,6 +98,11 @@ class ItemReport: # a failure: excluded from runs_passed and from pass@K, and pass^K is False while any # exist. Counted in `runs`, because they cost real agent time. runs_ungraded: int = 0 + # The agentic kinds' labels (``core/outcome.py``): ``harness_error`` is set only when every + # run was a harness error, ``shadow`` is the phase-2 candidate verdict. Neither decides + # anything here; they are reported for reading. + harness_error: dict | None = None + shadow: dict | None = None # What the item actually ran, when the kind knows better than the requested K. # agentic_conversation drives its fixture exactly once whatever --runs says, so # trusting K there reports four runs that never happened. diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index 09cfab387..9513102e3 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -26,7 +26,7 @@ render_turn_for_simulated_user, run_agentic_alert_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _DATE_FILTER = { @@ -1932,3 +1932,21 @@ def test_an_alert_created_before_the_stream_broke_is_still_cleaned_up(): assert mock_delete.call_count == 1 assert mock_delete.call_args.args[-1] == "alert-1" + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + ], +) +def test_exit_reason_names_the_agents_own_chat_failure(exc, exit_value): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = exc + + detail = _run_alert(mock_client, max_iterations=3) + + assert detail["exit_reason"] == exit_value + assert detail["alert_created"] is False diff --git a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py index 5c1e25d8b..4bdfe156b 100644 --- a/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py +++ b/packages/gooddata-eval/tests/test_agentic_anomaly_detection.py @@ -16,6 +16,7 @@ evaluate_agentic_anomaly_detection, run_agentic_anomaly_detection, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.anomaly_detection" @@ -431,7 +432,7 @@ def trace(self, _conversation_id): def observe(self, _trace, _run_idx, *, conversation_id=None): yield "trace-id" - def score(self, _tid, *, name, value, data_type): + def score(self, _tid, *, name, value, data_type, comment=None): self.scores[name] = value def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): @@ -539,3 +540,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index ca0f2dc2d..f08b0d63f 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -53,7 +53,7 @@ run_agentic_conversation, ) from gooddata_eval.core.chat.render import render_answer_text -from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.chat.sse_client import ChatError as ChatError_ from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent @@ -1209,6 +1209,32 @@ def test_a_partial_result_joins_the_shared_timeline_like_any_other_turn(): assert result.total_steps == 5 +def test_an_agent_timeout_ends_its_turn_with_its_own_exit_and_still_reads_as_an_error(): + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [ + _metric_turn_result([_skills_tc("metric"), _create_metric_tc("m1")]), + TurnTimeoutError("agent exceeded the 60s turn budget"), + ] + + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + ): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=_two_metric_turn_fixture(), + ) + + assert result.turn_results[0].exit_reason is LoopExit.SUCCESS + assert result.turn_results[1].exit_reason is LoopExit.AGENT_TIMEOUT + # The turn's no_error reads exactly as it did when a timeout was recorded as CHAT_ERROR. + assert result.turn_results[1].no_error is False + assert result.turn_results[1].output_present is False + + def test_a_non_chat_exception_still_propagates(): """Only chat faults are absorbed. A programming error must not be relabelled CHAT_ERROR.""" mock_client = MagicMock() diff --git a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py index 9b6075b9f..d053be358 100644 --- a/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_dashboard_skill.py @@ -20,7 +20,7 @@ evaluate_dashboard_response, run_agentic_dashboard_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent # Ids and titles are the ones the eval layout seeds; shapes are trimmed from real runs of @@ -2947,6 +2947,22 @@ def test_a_caller_supplied_conversation_is_not_deleted(self): client.create_conversation.assert_not_called() client.delete_conversation.assert_not_called() + @pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], + ) + def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(self, exc, loop_exit) -> None: + client = MagicMock() + client.send_message.side_effect = exc + + summary = _run_with(client, _DC05_EXPECTED) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + def test_a_chat_error_is_recorded_rather_than_left_to_escape(self) -> None: """It used to escape run_agentic_dashboard_skill, taking every completed K-run with it: no DashboardRunResult, no exit_reason, just a bare error on the item. Recorded diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index e1a0c0f2d..75f2a4e10 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -1724,15 +1724,22 @@ def test_the_last_document_part_of_a_turn_is_the_one_scored() -> None: class _FakeCtx: - """Records what the deferred Langfuse block writes, without a Langfuse.""" + """Records what the deferred Langfuse block writes, without a Langfuse. + + ``scores`` and ``comments`` hold every score whatever its run; ``run_scores`` and + ``run_comments`` hold them per conversation, for a test that cares which run got a score. + """ def __init__(self) -> None: self.run_metadata: dict[str, Any] = {} self.scores: dict[str, float] = {} + self.run_scores: dict[str, dict[str, float]] = {} + self.run_comments: dict[str, dict[str, str]] = {} self.score_types: dict[str, str] = {} self.quality_checks: dict[str, bool] = {} self.observed: list[str] = [] self.unscored: list[str] = [] + self.comments: dict[str, str] = {} def export_unscored(self, _run_idx: int, conversation_id: str) -> None: self.unscored.append(conversation_id) @@ -1741,13 +1748,19 @@ def trace(self, _conversation_id: str) -> None: return None @contextmanager - def observe(self, _trace: None, _run_idx: int, *, conversation_id: str, output: dict) -> Iterator[str]: + def observe( + self, _trace: None, _run_idx: int, *, conversation_id: str, output: dict | None = None + ) -> Iterator[str]: self.observed.append(conversation_id) - yield "trace-id" + yield conversation_id - def score(self, _tid: str, *, name: str, value: float, data_type: str) -> None: + def score(self, tid: str, *, name: str, value: float, data_type: str, comment: str | None = None) -> None: self.scores[name] = value + self.run_scores.setdefault(tid, {})[name] = value self.score_types[name] = data_type + if comment is not None: + self.comments[name] = comment + self.run_comments.setdefault(tid, {})[name] = comment def quality( self, _tid: str, *, strict_checks: dict[str, bool], latency_sec: float | None, cost_usd: float | None @@ -1991,12 +2004,16 @@ def test_an_ungraded_run_keeps_pass_at_k_but_not_pass_power_k(monkeypatch: pytes assert summary.best.judge_error is None -def test_an_ungraded_run_writes_no_scores(monkeypatch: pytest.MonkeyPatch) -> None: +def test_a_run_the_judge_could_not_grade_gets_only_the_harness_marker(monkeypatch: pytest.MonkeyPatch) -> None: ctx = _scored(monkeypatch, {"narrative": _NARRATIVE}, _narrative_turns(), k=2, judge=_SequenceJudge()) - assert ctx.observed == ["conv-2"] - assert ctx.unscored == ["conv-1"] - assert ctx.scores["document_narrative_judged"] == 1.0 - assert ctx.scores["document_summaries_present"] == 1.0 + # conv-1 has no verdict: its root carries eval_error alone; conv-2 gets its usual scores. + assert ctx.observed == ["conv-1", "conv-2"] + assert ctx.unscored == [] + assert ctx.run_scores["conv-1"] == {"eval_error": 1} + assert ctx.run_comments["conv-1"] == {"eval_error": "judge_failed"} + assert "eval_error" not in ctx.run_scores["conv-2"] + assert ctx.run_scores["conv-2"]["document_narrative_judged"] == 1.0 + assert ctx.run_scores["conv-2"]["document_summaries_present"] == 1.0 def _failing_narrative_turn() -> ChatResult: @@ -2009,8 +2026,13 @@ def test_a_run_that_failed_a_fixed_check_is_a_failure_even_when_the_judge_errors expected = {"narrative": _NARRATIVE, "visualizations": [{"id": _RETURNS_BY_CATEGORY, "title": "Returns"}]} _install_client(monkeypatch, _ScriptedChatClient([_failing_narrative_turn()])) judge = _FakeJudge(error=JudgeResponseError("empty body")) - with pytest.raises(DocumentSkillAssertionError, match="does not show chart 'returns_by_category'"): + with pytest.raises(DocumentSkillAssertionError, match="does not show chart 'returns_by_category'") as exc_info: evaluate_agentic_document_skill("https://h", "tok", "ws", "q", expected, judge=judge) + # Labelled a FAIL too: the judge breaking did not decide this run, the missing chart did. + exc = exc_info.value + assert [r["outcome"]["status"] for r in exc.failed_runs] == ["fail"] + assert not hasattr(exc, "harness_error") + assert "Harness error" not in str(exc) def test_a_failed_run_with_a_judge_error_is_published(monkeypatch: pytest.MonkeyPatch) -> None: @@ -2019,6 +2041,8 @@ def test_a_failed_run_with_a_judge_error_is_published(monkeypatch: pytest.Monkey assert ctx.observed == ["conv-1"] assert ctx.scores["document_charts_matched"] == 0.0 assert "document_narrative_judged" not in ctx.scores + # A FAIL, not a harness error: no marker. + assert "eval_error" not in ctx.run_scores["conv-1"] def test_only_content_pages_count_for_summaries() -> None: diff --git a/packages/gooddata-eval/tests/test_agentic_failure_message.py b/packages/gooddata-eval/tests/test_agentic_failure_message.py new file mode 100644 index 000000000..04b167bf0 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_failure_message.py @@ -0,0 +1,522 @@ +# (C) 2026 GoodData Corporation +"""The failing-runs head of an agentic failure message, and the per-run records it reads.""" + +import importlib +import inspect +from collections.abc import Callable +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import metric_skill as metric_skill_mod +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import ( + FailedCheck, + failed_check_records, + failing_runs_note, + failure_head, +) +from gooddata_eval.core.agentic._outcome import RunLabels +from gooddata_eval.core.agentic.alert_skill import AlertEvaluation +from gooddata_eval.core.agentic.alert_skill import _failed_checks as alert_failed_checks +from gooddata_eval.core.agentic.anomaly_detection import AnomalyEvaluation +from gooddata_eval.core.agentic.anomaly_detection import _failed_checks as anomaly_failed_checks +from gooddata_eval.core.agentic.general_question import _failed_checks as general_question_failed_checks +from gooddata_eval.core.agentic.guardrail import _failed_checks as guardrail_failed_checks +from gooddata_eval.core.agentic.kda_skill import KdaEvaluation +from gooddata_eval.core.agentic.kda_skill import _failed_checks as kda_failed_checks +from gooddata_eval.core.agentic.metric_skill import MetricSkillAssertionError, evaluate_agentic_metric_skill +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.agentic.what_if import WhatIfEvaluation +from gooddata_eval.core.agentic.what_if import _failed_checks as what_if_failed_checks +from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit +from gooddata_eval.core.outcome import Outcome, classify_run + +# ── the renderer ───────────────────────────────────────────────────────────── + + +def test_a_failed_check_records_only_what_the_kind_knows() -> None: + assert FailedCheck("metric_correct").as_record() == {"name": "metric_correct"} + assert FailedCheck("filters_correct", part="date", expected=[], actual=["MONTH -11..0"]).as_record() == { + "name": "filters_correct", + "part": "date", + "expected": [], + "actual": ["MONTH -11..0"], + } + # None is a real expected value, not "unknown". + assert FailedCheck("x", expected=None).as_record() == {"name": "x", "expected": None} + + +def test_the_note_lists_each_failing_run_with_its_exit_and_failing_checks_only() -> None: + failed_runs = [ + { + "run_index": 2, + "exit_reason": "success", + "failed_checks": failed_check_records( + [FailedCheck("filters_correct", part="date", expected=[], actual="MONTH -11..0")] + ), + }, + {"run_index": 3, "exit_reason": "budget_exhausted", "failed_checks": [{"name": "metric_correct"}]}, + ] + + note = failing_runs_note(failed_runs, 3) + + assert note == ( + "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual MONTH -11..0]; " + "run 3/3 FAIL (loop exit: budget_exhausted): metric_correct" + ) + + +def test_the_head_starts_with_the_gate_note() -> None: + gate_note = "Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure." + failed_runs = [{"run_index": 2, "exit_reason": "success", "failed_checks": [{"name": "metric_correct"}]}] + + head = failure_head(gate_note, failed_runs, 3) + + assert head.startswith(gate_note) + assert head.endswith("Failing runs: run 2/3 FAIL (loop exit: success): metric_correct.") + + +def test_the_head_is_the_gate_note_alone_without_failing_runs() -> None: + assert failure_head("Gate pass@2 failed: 0/2 runs passed.", [], 2) == "Gate pass@2 failed: 0/2 runs passed." + + +def test_a_run_without_recorded_checks_says_so_and_a_kind_without_the_key_says_nothing() -> None: + assert failing_runs_note([{"run_index": 1, "exit_reason": None, "failed_checks": []}], 1) == ( + "run 1/1 FAIL: no failing check recorded" + ) + assert failing_runs_note([{"run_index": 1, "exit_reason": "agent_silent"}], 1) == ( + "run 1/1 FAIL (loop exit: agent_silent)" + ) + + +def test_an_errored_run_is_labelled_with_its_reason_and_still_lists_its_checks() -> None: + record = { + "run_index": 1, + "exit_reason": "simulated_user_failed", + "outcome": {"status": "error", "reason": "simulated_user_failed"}, + "failed_checks": [{"name": "metric_created"}], + } + assert failing_runs_note([record], 3) == "run 1/3 ERROR (simulated_user_failed): metric_created" + + +def test_an_errored_run_without_failed_checks_is_its_label_alone() -> None: + record = {"run_index": 2, "outcome": {"status": "error", "reason": "judge_failed"}, "failed_checks": []} + assert failing_runs_note([record], 3) == "run 2/3 ERROR (judge_failed)" + + +def _long_run(run_index: int) -> dict: + checks = [{"name": f"check_{i}", "expected": "e" * 150, "actual": "a" * 150} for i in range(4)] + return {"run_index": run_index, "exit_reason": "budget_exhausted", "failed_checks": checks} + + +def test_failing_runs_past_the_cap_are_counted_not_rendered() -> None: + note = failing_runs_note([_long_run(i) for i in range(1, 6)], 5) + assert len(note) < 2100 + assert note.startswith("run 1/5 FAIL (loop exit: budget_exhausted): check_0[") + assert note.endswith("; 4 more failing run(s) in failed_runs") + + +def test_the_first_failing_run_is_shown_even_past_the_cap() -> None: + record = {"run_index": 1, "failed_checks": [{"name": f"check_{i}", "actual": "a" * 150} for i in range(30)]} + note = failing_runs_note([record], 1) + assert note.startswith("run 1/1 FAIL: check_0[") + assert note.endswith("...") + assert len(note) == 2000 + + +def test_a_long_value_is_cut_to_one_short_line() -> None: + record = {"run_index": 1, "failed_checks": [{"name": "maql_correct", "actual": "SELECT\n" + "x" * 500}]} + note = failing_runs_note([record], 1) + assert "\n" not in note + assert note.endswith("...]") + assert len(note) < 250 + + +# ── the records ────────────────────────────────────────────────────────────── + + +@dataclass +class _Run: + ok: bool + exit_reason: LoopExit = LoopExit.SUCCESS + conversation_id: str = "c" + tool_call_events: list = field(default_factory=list) + + +def test_every_failing_record_carries_its_exit_reason_and_failed_checks() -> None: + runs = [_Run(True), _Run(False, LoopExit.BUDGET_EXHAUSTED)] + + records = build_failed_runs( + runs, + passed=lambda r: r.ok, + detail=lambda r: {}, + failed_checks=lambda r: [{"name": "metric_correct"}], + ) + + assert len(records) == 1 + assert records[0]["run_index"] == 2 + assert records[0]["exit_reason"] == "budget_exhausted" + assert records[0]["failed_checks"] == [{"name": "metric_correct"}] + + +def test_a_kind_that_names_no_checks_writes_no_failed_checks_key() -> None: + records = build_failed_runs([object()], passed=lambda r: False, detail=lambda r: {}) + assert records[0]["exit_reason"] is None + assert "failed_checks" not in records[0] + + +@dataclass +class _StrRun: + exit_reason: str = "chat_error" + + +def test_a_string_exit_reason_gets_the_same_outcome_as_its_enum() -> None: + # exit_reason and outcome must agree whether a kind records the member or its value. + for exit_reason in (LoopExit.SIMULATED_USER_FAILED, "simulated_user_failed"): + record = build_failed_runs([_StrRun(exit_reason)], passed=lambda r: False, detail=lambda r: {})[0] + assert record["exit_reason"] == "simulated_user_failed" + assert record["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + + +@pytest.mark.parametrize( + "exit_reason", [LoopExit.CHAT_ERROR, "chat_error", "budget_exhausted"], ids=["member", "value", "agent_exit"] +) +def test_a_chat_error_or_agent_exit_is_recorded_and_rendered_as_a_failure(exit_reason: LoopExit | str) -> None: + records = build_failed_runs( + [_StrRun(exit_reason)], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: [] + ) + + value = LoopExit(exit_reason).value + assert records[0]["exit_reason"] == value + assert records[0]["outcome"] == {"status": "fail", "reason": value} + assert failing_runs_note(records, 3) == f"run 1/3 FAIL (loop exit: {value}): no failing check recorded" + + +def test_a_simulated_user_failure_is_recorded_and_rendered_as_an_error() -> None: + records = build_failed_runs( + [_StrRun("simulated_user_failed")], passed=lambda r: False, detail=lambda r: {}, failed_checks=lambda r: [] + ) + + assert records[0]["outcome"]["status"] == "error" + assert failing_runs_note(records, 3) == "run 1/3 ERROR (simulated_user_failed)" + + +@dataclass +class _JudgedRun: + judge_error: str | None + exit_reason: str = "success" + + +def test_a_run_the_judge_could_not_grade_is_recorded_as_an_error() -> None: + (record,) = build_failed_runs([_JudgedRun("empty body twice")], passed=lambda r: False, detail=lambda r: {}) + assert record["outcome"] == {"status": "error", "reason": "judge_failed"} + + +def test_a_labelled_kind_records_the_outcome_its_labels_give() -> None: + """The labels decide both which runs are recorded and their outcome: a run whose judge broke + after it failed a fixed check is labelled FAIL, and its record says so, judge_error or not.""" + runs = [_JudgedRun(None), _JudgedRun("empty body twice"), _JudgedRun(None, exit_reason="simulated_user_failed")] + labels = RunLabels((Outcome.passed(), Outcome.failed("success"), Outcome.error("simulated_user_failed"))) + + records = build_failed_runs(runs, labels=labels, detail=lambda r: {}) + + assert [r["run_index"] for r in records] == [2, 3] + assert records[0]["error"] == "empty body twice" + assert records[0]["outcome"] == {"status": "fail", "reason": "success"} + assert records[1]["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + + +@pytest.mark.parametrize( + "kwargs", + [{}, {"passed": lambda r: False, "labels": RunLabels((Outcome.failed(),))}], + ids=["neither", "both"], +) +def test_failed_runs_take_exactly_one_of_labels_and_passed(kwargs: dict[str, Any]) -> None: + with pytest.raises(ValueError, match="exactly one"): + build_failed_runs([_JudgedRun(None)], detail=lambda r: {}, **kwargs) + + +def test_failed_runs_refuse_labels_for_a_different_number_of_runs() -> None: + with pytest.raises(ValueError, match="1 labels for 2 runs"): + build_failed_runs([_JudgedRun(None)] * 2, labels=RunLabels((Outcome.failed(),)), detail=lambda r: {}) + + +def test_a_failed_check_record_keeps_short_values_and_caps_long_ones() -> None: + long_actual = {"filters": ["x" * 400]} + records = build_failed_runs( + [_StrRun("budget_exhausted")], + passed=lambda r: False, + detail=lambda r: {"actual": long_actual}, + failed_checks=lambda r: [ + {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]}, + {"name": "filters_correct", "part": "attribute", "expected": [], "actual": long_actual}, + ], + ) + + short, long_check = records[0]["failed_checks"] + assert short == {"name": "metrics_correct", "expected": ["metric/a"], "actual": ["metric/b"]} + assert long_check["expected"] == [] + assert isinstance(long_check["actual"], str) + assert len(long_check["actual"]) == 160 + assert long_check["actual"].endswith("...") + # The run's detail keeps the whole value. + assert records[0]["detail"]["actual"] == long_actual + + +def test_a_fail_run_names_its_loop_exit_as_its_reason() -> None: + assert classify_run(LoopExit.SUCCESS, passed=False).reason == "success" + + +def test_a_metric_layout_failure_names_the_layout_problem_not_two_equal_sets() -> None: + ev = MagicMock( + scored_without_chart=False, + visualization_created=True, + cross_ref_valid=True, + metrics_correct=False, + expected_metric_uris={"metric/revenue"}, + actual_metric_uris={"metric/revenue"}, + metric_layout_problems=["metric/revenue is on the secondary axis"], + dimensions_correct=True, + filters_correct=True, + sorts_correct=True, + viz_type_hard=True, + requires_execution=False, + buckets_correct=None, + answer_correct=None, + format_correct=None, + part_present=None, + ) + run = MagicMock(eval_result=ev) + + assert agentic_visualization._failed_checks(run) == [ + {"name": "metrics_correct", "part": "layout", "actual": ["metric/revenue is on the secondary axis"]} + ] + + +# ── visualization: a partial gate failure names the run that failed ────────── + + +def _viz(filter_by: dict | None = None) -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": filter_by or {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn(filter_by: dict | None = None) -> ChatResult: + return ChatResult.model_validate( + {"createdVisualizations": {"objects": [_viz(filter_by)], "reasoning": ""}, "toolCallEvents": []} + ) + + +_LAST_YEAR = {"date": {"type": "date_filter", "using": "dataset/date", "granularity": "MONTH", "from": -11, "to": 0}} + + +def test_a_partial_visualization_failure_lists_the_failing_run_and_keeps_the_best_run_summary() -> None: + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [_viz_turn(), _viz_turn(_LAST_YEAR), _viz_turn()] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=3, + gate="power", + ) + + message = str(exc_info.value) + lines = message.splitlines() + gate_line = next(line for line in lines if line.startswith("Gate pass^3 failed")) + assert gate_line.startswith("Gate pass^3 failed: 2/3 runs passed — unstable, not a clean failure. Failing runs: ") + assert "run 2/3 FAIL (loop exit: success): filters_correct[date: expected none, actual " in gate_line + # Only the failing checks of the failing run, and no passing run. + assert "metrics_correct" not in gate_line + assert "run 1/3" not in gate_line + assert "run 3/3" not in gate_line + # The best-run summary that followed the gate note is still there, unchanged. + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + assert "Strict Check Summary:" in message + assert " Filters Correct : True\n" in message + + (record,) = exc_info.value.failed_runs + assert record["run_index"] == 2 + assert record["exit_reason"] == "success" + assert record["detail"]["exit_reason"] == "success" + assert [c["name"] for c in record["failed_checks"]] == ["filters_correct"] + assert record["failed_checks"][0]["part"] == "date" + assert record["failed_checks"][0]["expected"] == [] + + +# ── metric skill: same head, expected/actual MAQL of the failing run ───────── + + +def _metric_turn(maql: str) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": f'{{"data": {{"maql": "{maql}"}}}}', + } + ], + } + ) + + +def test_a_partial_metric_failure_lists_the_failing_run_with_its_maql() -> None: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = [_metric_turn("SELECT {metric/foo}"), _metric_turn("SELECT {metric/bar}")] + + with ( + patch.object(metric_skill_mod, "ChatClient", return_value=client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + gate="power", + ) + + message = str(exc_info.value) + assert message.startswith( + "Metric skill assertion failed. Gate pass^2 failed: 1/2 runs passed — unstable, not a clean failure. " + "Failing runs: run 2/2 FAIL (loop exit: success): " + 'maql_correct[expected ["SELECT {metric/foo}"], actual SELECT {metric/bar}].' + ) + # The best run's summary follows the failing runs. + assert "metric_created=True, maql_correct=True. Expected MAQL (candidates): 'SELECT {metric/foo}'." in message + (record,) = exc_info.value.failed_runs + assert record["exit_reason"] == "success" + assert record["failed_checks"] == [ + {"name": "maql_correct", "expected": ["SELECT {metric/foo}"], "actual": "SELECT {metric/bar}"} + ] + + +def test_a_run_that_created_no_metric_names_only_the_missing_metric() -> None: + """Without a metric, maql_correct reads False only because there is no MAQL to compare.""" + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = ChatResult.model_validate({"textResponse": "I cannot do that."}) + + with ( + patch.object(metric_skill_mod, "ChatClient", return_value=client), + pytest.raises(MetricSkillAssertionError) as exc_info, + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=1, + gate="power", + ) + + (record,) = exc_info.value.failed_runs + assert record["failed_checks"] == [{"name": "metric_created", "expected": True, "actual": False}] + assert "maql_correct[" not in str(exc_info.value) + + +@pytest.mark.parametrize("module_name", ["visualization", "alert_skill", "metric_skill", "dashboard_skill"]) +def test_each_checked_kind_heads_its_message_through_the_shared_renderer(module_name: str) -> None: + source = inspect.getsource(importlib.import_module(f"gooddata_eval.core.agentic.{module_name}")) + assert "failure_message_head(gate, run_labels, runs_passed, failed_runs)" in source + assert "failed_checks=_failed_checks" in source + + +# ── the kinds that name their failing checks by name only ───────────────────── + + +def test_the_completion_kinds_name_exactly_the_checks_strict_pass_failed() -> None: + kda = KdaEvaluation(triggered=True, executed=False, success=False, turn_completed=True) + assert kda_failed_checks(SimpleNamespace(evaluation=kda)) == [{"name": "executed"}, {"name": "success"}] + + anomaly = AnomalyEvaluation( + triggered=True, + executed=True, + success=True, + turn_completed=True, + metric_correct=False, + granularity_correct=True, + ) + assert anomaly_failed_checks(SimpleNamespace(evaluation=anomaly)) == [{"name": "metric_correct"}] + assert not anomaly.strict_pass + + what_if = WhatIfEvaluation( + triggered=True, + executed=True, + success=True, + turn_completed=True, + metric_correct=True, + maql_correct=True, + scenario_count_correct=True, + baseline_correct=True, + ) + assert what_if.strict_pass + assert what_if_failed_checks(SimpleNamespace(evaluation=what_if)) == [] + + +def _alert_evaluation(created: bool, **checks: bool) -> AlertEvaluation: + fields = dict.fromkeys( + ( + "operator_correct", + "threshold_correct", + "trigger_correct", + "filters_correct", + "metric_correct", + "recipients_correct", + ), + created, + ) + return AlertEvaluation(alert_created=created, **{**fields, **checks}) + + +def test_an_alert_run_that_created_no_alert_names_only_the_missing_alert() -> None: + """Every other alert check reads False when nothing was created; naming them all would + report argument errors for an alert that does not exist.""" + run = SimpleNamespace(eval=_alert_evaluation(False, title_correct=False)) + assert alert_failed_checks(run) == [{"name": "alert_created"}] + + +def test_an_alert_run_that_created_an_alert_names_each_check_it_failed() -> None: + run = SimpleNamespace(eval=_alert_evaluation(True, threshold_correct=False, title_correct=False)) + assert alert_failed_checks(run) == [{"name": "threshold_correct"}, {"name": "title_correct"}] + + +@pytest.mark.parametrize( + ("failed_checks", "name"), + [(general_question_failed_checks, "general_question_pass"), (guardrail_failed_checks, "guardrail_pass")], +) +def test_a_judged_kind_names_the_judges_reasoning_and_nothing_for_an_ungraded_run( + failed_checks: Callable[[Any], list[dict]], name: str +) -> None: + failed = SimpleNamespace(judge_error=None, passed=False, reasoning="answered a different question") + assert failed_checks(failed) == [{"name": name, "part": "judge", "actual": "answered a different question"}] + ungraded = SimpleNamespace(judge_error="empty body", passed=False, reasoning="") + assert failed_checks(ungraded) == [] diff --git a/packages/gooddata-eval/tests/test_agentic_gate.py b/packages/gooddata-eval/tests/test_agentic_gate.py index e79af9093..bad390e57 100644 --- a/packages/gooddata-eval/tests/test_agentic_gate.py +++ b/packages/gooddata-eval/tests/test_agentic_gate.py @@ -99,6 +99,22 @@ def test_failure_note_blames_the_judge_not_the_agent_for_an_ungraded_run(): assert "unstable" not in note +def test_failure_note_names_harness_errors_without_calling_them_instability(): + note = gate_failure_note("power", 2, 3, runs_errored=1) + assert note == "Gate pass^3 failed: 2/3 runs passed, 1 harness error(s)." + + +def test_failure_note_keeps_an_agent_failure_unstable_beside_a_harness_error(): + """Pass, fail, error: the errored run is named, but the run the agent failed beside a + passing one is still instability.""" + note = gate_failure_note("power", 1, 3, runs_errored=1) + assert note == "Gate pass^3 failed: 1/3 runs passed, 1 harness error(s) — unstable, not a clean failure." + + +def test_failure_note_does_not_call_errored_runs_alone_unstable_under_pass_at_k(): + assert gate_failure_note("any", 0, 3, runs_errored=1) == "Gate pass@3 failed: 0/3 runs passed, 1 harness error(s)." + + def test_gate_scores_use_K_stable_names(): """`pass_at_2` becomes `pass_at_3` the moment K changes, splitting every view on it.""" ctx = MagicMock() diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 6a1690ebb..8f13a4093 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -16,8 +16,8 @@ evaluate_agentic_kda_skill, run_agentic_kda_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError -from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.models import ChatResult, LoopExit _EXPECTED = {"Measure": {"type": "metric", "id": "revenue"}} @@ -1296,3 +1296,29 @@ def _capture(_submit, _identity, **kwargs): assert scores["turns"] == 1 assert scores["steps"] == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("gen-ai returned 503"), LoopExit.CHAT_ERROR), + ], +) +def test_run_agentic_kda_skill_names_why_the_chat_call_failed(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=1, + ) + + assert summary.best.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 5efcbb0bb..b8ebb0c63 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -22,7 +22,7 @@ generate_simulated_response, run_agentic_metric_skill, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent from gooddata_eval.core.timing import TIMERS_ENV_VAR @@ -1098,3 +1098,81 @@ def call(mid: str, created_new: bool) -> ToolCallEvent: ) assert _extract_created_metric_ids([call("new", True), call("old", False)]) == ["new"] + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + mock_client = _client() + mock_client.send_message.side_effect = exc + + with _patched(mock_client): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + max_iterations=3, + ) + + assert summary.run_results[0].exit_reason is loop_exit + assert summary.pass_at_k is False + + +def test_a_chat_error_run_is_a_failure_scored_as_before(): + mock_client = _client() + mock_client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate( + { + "textResponse": "done", + "toolCallEvents": [ + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": '{"data": {"maql": "SELECT {metric/foo}"}}', + } + ], + } + ), + ] + captured = {} + + def _capture(_submit, _identity, **kwargs): + captured["write_scores"] = kwargs["write_scores"] + + with ( + _patched(mock_client), + patch("gooddata_eval.core.agentic.metric_skill.submit_trace_scoring", _capture), + ): + evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + + ctx = MagicMock() + run_ids = [MagicMock(name="run-1"), MagicMock(name="run-2")] + ctx.observe.return_value.__enter__.side_effect = run_ids + captured["write_scores"](ctx) + by_run = {id(t): [c.kwargs["name"] for c in ctx.score.call_args_list if c.args[0] is t] for t in run_ids} + + first, second = by_run[id(run_ids[0])], by_run[id(run_ids[1])] + # A chat API error is on the product path: a FAIL, with no harness marker. + assert "eval_error" not in first + assert "gate_passed" in first + assert "eval_error" not in second + assert "gate_passed" in second diff --git a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py index 83f31a277..44a4d4fba 100644 --- a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py +++ b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py @@ -345,3 +345,15 @@ def test_the_run_context_forwards_its_window_and_item_input_to_observe(): "item_input": "Show revenue by month", "output": {"passed": True}, } + + +def test_the_run_context_forwards_a_score_comment_only_when_given(): + lf = MagicMock() + ctx = RunTraceContext({}, lf, "client", "item-1", "base", True, {}, _WINDOW, "q") + + ctx.score("tid", name="gate_passed", value=True, data_type="BOOLEAN") + ctx.score("tid", name="eval_error", value=1, data_type="NUMERIC", comment="backend: the chat call failed") + + plain, commented = lf.score_safe.call_args_list + assert plain.kwargs == {"name": "gate_passed", "value": True, "data_type": "BOOLEAN"} + assert commented.kwargs["comment"] == "backend: the chat call failed" diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index a3653f853..26abd44e2 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -22,7 +22,8 @@ from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.langfuse.item_scope import ItemTraceScope, current_scope -from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, CreatedVisualization, DatasetItem +from gooddata_eval.core.reporting.json_report import build_json_report from gooddata_eval.core.timing import PhaseTimings, note_trace_latency @@ -1022,6 +1023,37 @@ def test_a_kind_that_records_no_failed_runs_keeps_the_empty_default(): assert report.items[0].failed_runs == [] +_HARNESS_ERROR = {"status": "error", "reason": "simulated_user_failed"} +_SHADOW = {"verdict": "error", "runs_scored": 0, "runs_total": 3} + + +def _labelled_failure(exc: AgenticAssertionError | JudgeResponseError) -> Exception: + exc.harness_error = _HARNESS_ERROR + exc.shadow = _SHADOW + return exc + + +@pytest.mark.parametrize( + "result", + [ + pytest.param(AgenticEvalOutcome(reasoning_steps=[], detail={}, shadow=_SHADOW), id="passed"), + pytest.param(_labelled_failure(AlertSkillAssertionError("failed")), id="failed"), + pytest.param(_labelled_failure(JudgeResponseError("no verdict")), id="ungraded"), + ], +) +def test_the_item_labels_reach_the_report(result: Any): + """A label nobody reads labels nothing: the outcome and the shadow verdict travel from + whichever object the kind returned or raised onto the item report and into the JSON.""" + kwargs = {"return_value": result} if isinstance(result, AgenticEvalOutcome) else {"side_effect": result} + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", **kwargs): + report = run_agentic_items([_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") + item = report.items[0] + assert item.shadow == _SHADOW + assert item.harness_error == (None if isinstance(result, AgenticEvalOutcome) else _HARNESS_ERROR) + record = build_json_report(report)["items"][item.id] + assert (record["harness_error"], record["shadow"]) == (item.harness_error, item.shadow) + + def test_an_all_ungraded_item_is_errored_but_still_carries_its_failed_runs(): """JudgeResponseError is a RuntimeError, so it used to land in the generic branch: errored, runs=0, no records. It is still an error -- no run has a verdict -- but the diff --git a/packages/gooddata-eval/tests/test_agentic_visualization.py b/packages/gooddata-eval/tests/test_agentic_visualization.py index 431c3f329..c50d2fddb 100644 --- a/packages/gooddata-eval/tests/test_agentic_visualization.py +++ b/packages/gooddata-eval/tests/test_agentic_visualization.py @@ -18,7 +18,7 @@ evaluate_agentic_visualization, run_agentic_visualization, ) -from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.chat.sse_client import ChatError, TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.evaluators.visualization import ExpectationOptions from gooddata_eval.core.models import ChatResult, CreatedVisualization, LoopExit @@ -822,3 +822,35 @@ def test_a_failed_request_with_a_cut_off_answer_fails_an_item_without_a_chart() assert result.exit_reason is LoopExit.CHAT_ERROR assert result.eval_result.strict_pass is False assert result.eval_result.answer_problems == ["the chat request failed before the answer ended"] + + +_AGENT_CHAT_FAILURES = [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), +] + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_the_opening_request(exc, loop_exit): + client = MagicMock() + client.send_message.side_effect = exc + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit + assert result.eval_result.visualization_created is False + # Like a chat error, the loop does not run after the opening failure. + assert client.send_message.call_count == 1 + + +@pytest.mark.parametrize(("exc", "loop_exit"), _AGENT_CHAT_FAILURES) +def test_execute_single_run_records_the_agents_own_failure_on_a_follow_up(monkeypatch, exc, loop_exit): + monkeypatch.setattr( + "gooddata_eval.core.agentic.visualization.generate_simulated_response", lambda *a, **k: "the revenue one" + ) + client = MagicMock() + client.send_message.side_effect = [_chat_clarification(), exc] + + result = _execute_single_run(client, "conv-1", "Show revenue", [_expected()]) + + assert result.exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_agentic_what_if.py b/packages/gooddata-eval/tests/test_agentic_what_if.py index 38b04ebf3..4ee0e4ab0 100644 --- a/packages/gooddata-eval/tests/test_agentic_what_if.py +++ b/packages/gooddata-eval/tests/test_agentic_what_if.py @@ -13,6 +13,7 @@ evaluate_agentic_what_if, run_agentic_what_if, ) +from gooddata_eval.core.chat.sse_client import TurnIncompleteError, TurnTimeoutError from gooddata_eval.core.models import ChatResult, LoopExit _MODULE = "gooddata_eval.core.agentic.what_if" @@ -385,7 +386,7 @@ def trace(self, _conversation_id): def observe(self, _trace, _run_idx, *, conversation_id=None): yield "trace-id" - def score(self, _tid, *, name, value, data_type): + def score(self, _tid, *, name, value, data_type, comment=None): self.scores[name] = value def quality(self, _tid, *, strict_checks, latency_sec, cost_usd): @@ -499,3 +500,15 @@ def test_any_gate_still_passes_the_same_item(): def test_the_default_gate_is_pass_at_k(): assert _two_runs().runs_passed == 1 + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + ], +) +def test_the_agents_own_chat_failure_is_recorded_as_its_own_exit(exc, loop_exit): + summary = _run([_chat(_pair()), exc], k=2) + assert summary.run_results[1].exit_reason is loop_exit diff --git a/packages/gooddata-eval/tests/test_eval_error_scores.py b/packages/gooddata-eval/tests/test_eval_error_scores.py new file mode 100644 index 000000000..38ceab535 --- /dev/null +++ b/packages/gooddata-eval/tests/test_eval_error_scores.py @@ -0,0 +1,162 @@ +# (C) 2026 GoodData Corporation +"""Every agentic loop kind writes its Langfuse scores the same way. + +A run labelled ERROR gets an extra ``eval_error`` score beside its usual ones; every other run, +a chat API error included, gets its usual scores and no ``eval_error``. The visualization kind +has its own tests in test_outcome.py. +""" + +from collections import defaultdict +from collections.abc import Callable, Iterator +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import alert_skill, anomaly_detection, dashboard_skill, kda_skill, what_if +from gooddata_eval.core.agentic._outcome import RunLabels +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.outcome import Outcome + + +@dataclass(frozen=True) +class _Kind: + module: Any + evaluate: Callable[..., Any] + expected_output: dict + simulated_user: str | None + + +_KINDS = { + "alert_skill": _Kind( + alert_skill, + alert_skill.evaluate_agentic_alert_skill, + {"operator": "GREATER_THAN", "threshold": 100}, + "generate_simulated_alert_response", + ), + "kda_skill": _Kind( + kda_skill, + kda_skill.evaluate_agentic_kda_skill, + {"Measure": {"type": "metric", "id": "revenue"}}, + "generate_simulated_kda_response", + ), + "dashboard_skill": _Kind( + dashboard_skill, + dashboard_skill.evaluate_agentic_dashboard_skill, + { + "type": "dashboard", + "saved_dashboard_id": None, + "visualizations": [{"id": None, "title": "Revenue"}], + "date_range": None, + "min_new_visualizations": 1, + }, + None, + ), + "anomaly_detection": _Kind( + anomaly_detection, + anomaly_detection.evaluate_agentic_anomaly_detection, + {"metric": "metric/spend", "granularity": "MONTH"}, + "generate_simulated_anomaly_response", + ), + "what_if": _Kind( + what_if, + what_if.evaluate_agentic_what_if, + {"metric_id": "revenue"}, + "generate_simulated_what_if_response", + ), +} + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + def run_name(self, run_idx: int) -> str: + return f"run{run_idx}" + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(kind: _Kind, labels: RunLabels | None = None) -> _ScoreCtx: + """Run 1's chat call fails after its retries; run 2 answers without doing the task. + + ``labels`` replaces the labels the kind computes, to give run 1 a harness ERROR without + driving each kind's own simulated-user loop.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2", "conv-3"] + client.send_message.side_effect = [ + ChatError("gen-ai fell over"), + ChatResult.model_validate({"textResponse": "I cannot help with that."}), + ] + captured: dict[str, Any] = {} + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + captured["write_scores"] = kwargs["write_scores"] + + with ExitStack() as stack: + stack.enter_context(patch.object(kind.module, "ChatClient", return_value=client)) + stack.enter_context(patch.object(kind.module, "submit_trace_scoring", _capture)) + if labels is not None: + stack.enter_context(patch.object(kind.module, "label_runs", return_value=labels)) + if kind.simulated_user is not None: + stack.enter_context(patch.object(kind.module, kind.simulated_user, return_value="please go on")) + try: + kind.evaluate( + "http://host/api/v1/actions/workspaces/ws1/ai", + "tok", + "ws1", + "Do the thing", + kind.expected_output, + k=2, + max_iterations=1, + langfuse=MagicMock(), + dataset_item_id="item-1", + ) + except AssertionError: + pass # the scored run fails; its scores are written before the raise + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +_USUAL_SCORES = {"pass_at_k", "pass_power_k", "gate_passed"} + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_an_errored_run_gets_eval_error_beside_its_usual_scores(name: str) -> None: + labels = RunLabels((Outcome.error("simulated_user_failed"), Outcome.failed("agent_silent"))) + ctx = _scores(_KINDS[name], labels) + + assert ("eval_error", 1, "simulated_user_failed") in ctx.scores[0] + assert set(ctx.names(0)) >= _USUAL_SCORES + assert "eval_error" not in ctx.names(1) + assert set(ctx.names(1)) >= _USUAL_SCORES + + +@pytest.mark.parametrize("name", sorted(_KINDS)) +def test_a_chat_error_run_is_scored_as_before_with_no_marker(name: str) -> None: + ctx = _scores(_KINDS[name]) + + for run_idx in (0, 1): + names = ctx.names(run_idx) + assert "eval_error" not in names + assert set(names) >= _USUAL_SCORES diff --git a/packages/gooddata-eval/tests/test_outcome.py b/packages/gooddata-eval/tests/test_outcome.py new file mode 100644 index 000000000..0fb00a545 --- /dev/null +++ b/packages/gooddata-eval/tests/test_outcome.py @@ -0,0 +1,539 @@ +# (C) 2026 GoodData Corporation +"""PASS / FAIL / ERROR labels per run: they explain, they never rescore. + +A run is ERROR only when the harness failed -- its simulated user, or its judge when the judge's +verdict was the one still open. Every other non-pass, a chat API error and the agent's own +timeout included, is FAIL. The labels change no gate or count: an ERROR run still counts as a +run that did not pass. The message names the ERROR runs and counts them in its gate note. +""" + +from collections import defaultdict +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic._gate import gate_failure_note +from gooddata_eval.core.agentic._outcome import ( + RunLabels, + chat_failure_exit, + failure_message_head, + label_runs, + mark_unscored_run, +) +from gooddata_eval.core.agentic.document_skill import _judge_error_of +from gooddata_eval.core.agentic.visualization import VisualizationAssertionError, evaluate_agentic_visualization +from gooddata_eval.core.chat.sse_client import ChatError, TransientChatError, TurnIncompleteError, TurnTimeoutError +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + CreatedVisualization, + LoopExit, +) +from gooddata_eval.core.outcome import ( + JUDGE_FAILED, + Outcome, + OutcomeStatus, + classify_run, + item_outcome, + shadow_verdict, +) + +PASS = Outcome.passed() +FAIL = Outcome.failed("agent_silent") +SIM = Outcome.error("simulated_user_failed") +JUDGE = Outcome.error(JUDGE_FAILED) + + +# ── the rule: an allowlist of harness causes, everything else is FAIL ───────── + +# Every LoopExit, with the label of a run that ended on it without passing. Exhaustive on +# purpose: a new member must be classified here before the suite passes. +_EXPECTED = { + LoopExit.SUCCESS: OutcomeStatus.FAIL, + LoopExit.AGENT_SILENT: OutcomeStatus.FAIL, + LoopExit.BUDGET_EXHAUSTED: OutcomeStatus.FAIL, + LoopExit.STOPPED_EARLY: OutcomeStatus.FAIL, + LoopExit.SIMULATED_USER_FAILED: OutcomeStatus.ERROR, + LoopExit.CHAT_ERROR: OutcomeStatus.FAIL, + LoopExit.AGENT_TIMEOUT: OutcomeStatus.FAIL, + LoopExit.TURN_INCOMPLETE: OutcomeStatus.FAIL, + LoopExit.NOT_RUN: OutcomeStatus.FAIL, +} + + +def test_every_loop_exit_is_classified() -> None: + assert set(_EXPECTED) == set(LoopExit) + + +@pytest.mark.parametrize("loop_exit", list(LoopExit)) +def test_each_loop_exit_has_its_label_and_names_itself_as_the_reason(loop_exit: LoopExit) -> None: + outcome = classify_run(loop_exit, passed=False) + assert outcome.status is _EXPECTED[loop_exit] + assert outcome.reason == loop_exit.value + assert classify_run(loop_exit.value, passed=False) == outcome + + +@pytest.mark.parametrize("loop_exit", list(LoopExit)) +def test_a_run_that_passed_is_pass_whatever_its_exit(loop_exit: LoopExit) -> None: + assert classify_run(loop_exit, passed=True) == PASS + + +def test_a_judge_failure_is_an_error_whatever_the_exit() -> None: + assert classify_run(LoopExit.SUCCESS, passed=False, judge_error="empty body twice") == JUDGE + assert classify_run(None, passed=False, judge_error="unparseable JSON") == JUDGE + + +def test_an_unknown_cause_is_a_failure_never_an_error() -> None: + assert classify_run("some_new_exit", passed=False) == Outcome.failed("some_new_exit") + assert classify_run(None, passed=False) == Outcome.failed(None) + + +def test_an_outcome_record_is_json_ready() -> None: + assert SIM.as_record() == {"status": "error", "reason": "simulated_user_failed"} + assert Outcome.failed("chat_error").as_record() == {"status": "fail", "reason": "chat_error"} + + +@pytest.mark.parametrize( + ("exc", "loop_exit"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), LoopExit.AGENT_TIMEOUT), + (TurnIncompleteError("no final answer", reason="max iterations"), LoopExit.TURN_INCOMPLETE), + (TransientChatError("503 after retries"), LoopExit.CHAT_ERROR), + (ChatError("SSE stream error"), LoopExit.CHAT_ERROR), + (httpx.ReadError("connection reset"), LoopExit.CHAT_ERROR), + ], +) +def test_a_chat_failure_maps_to_its_own_exit_and_is_a_failure(exc: BaseException, loop_exit: LoopExit) -> None: + assert chat_failure_exit(exc) is loop_exit + assert classify_run(chat_failure_exit(exc), passed=False).status is OutcomeStatus.FAIL + + +# ── the item: ERROR only when no run was scored ────────────────────────────── + + +def test_an_item_is_an_error_only_when_every_run_errored() -> None: + assert item_outcome([SIM, SIM, SIM]) == SIM + assert item_outcome([SIM, PASS, PASS]) is None + assert item_outcome([SIM, FAIL]) is None + assert item_outcome([]) is None + + +def test_an_errored_item_names_its_most_frequent_reason() -> None: + assert item_outcome([SIM, JUDGE, JUDGE]) == JUDGE + + +# ── the shadow verdict: the phase-2 candidate rule, recorded and never used ── + + +@pytest.mark.parametrize( + ("outcomes", "gate", "verdict", "scored"), + [ + ([SIM, PASS, PASS], "power", "pass", 2), + ([SIM, SIM, PASS], "power", "error", 1), # fewer than ceil(3/2) scored runs + ([SIM, SIM, SIM], "power", "error", 0), + ([Outcome.failed("chat_error"), PASS, PASS], "power", "fail", 3), # a chat error is scored + ([SIM, PASS], "power", "pass", 1), # K=2 needs one scored run + ([SIM], "power", "error", 0), + ([SIM, FAIL, PASS], "any", "pass", 2), + ([PASS, PASS, PASS], "power", "pass", 3), + ], +) +def test_the_shadow_verdict(outcomes: list[Outcome], gate: str, verdict: str, scored: int) -> None: + assert shadow_verdict(outcomes, gate) == {"verdict": verdict, "runs_scored": scored, "runs_total": len(outcomes)} + + +class _Run: + def __init__(self, ok: bool, exit_reason: LoopExit | None = None, judge_error: str | None = None) -> None: + self.ok = ok + self.exit_reason = exit_reason + self.judge_error = judge_error + + +def test_label_runs_reads_each_runs_exit_and_judge() -> None: + runs = [ + _Run(False, LoopExit.SIMULATED_USER_FAILED), + _Run(False, LoopExit.SUCCESS, judge_error="no verdict"), + _Run(True, LoopExit.SUCCESS), + _Run(False, LoopExit.CHAT_ERROR), + ] + labels = label_runs(runs, passed=lambda r: r.ok) + assert labels.outcomes == (SIM, JUDGE, PASS, Outcome.failed("chat_error")) + assert labels.error_label(0) == SIM + assert labels.error_label(2) is None + assert labels.error_label(3) is None + assert labels.error_label(9) is None + assert labels.item_outcome is None + assert labels.shadow("power") == {"verdict": "fail", "runs_scored": 2, "runs_total": 4} + + +def test_a_kind_can_name_its_own_judge_error() -> None: + run = _Run(False, LoopExit.SUCCESS, judge_error="takeaway request failed") + assert label_runs([run], passed=lambda r: r.ok).outcomes == (JUDGE,) + labels = label_runs([run], passed=lambda r: r.ok, judge_error=lambda r: None) + assert labels.outcomes == (Outcome.failed("success"),) + + +class _Evaluation: + def __init__(self, judge_error: str | None, fill_error: str | None, checks_held: bool) -> None: + self.judge_error = judge_error + self.fill_error = fill_error + self.ungraded = (judge_error is not None or fill_error is not None) and checks_held + + +class _DocRun: + def __init__(self, judge_error: str | None, fill_error: str | None, checks_held: bool = True) -> None: + self.evaluation = _Evaluation(judge_error, fill_error, checks_held) + self.exit_reason = None + + +def test_a_document_run_whose_takeaway_request_failed_is_a_failure_not_a_judge_error() -> None: + request_failed = _DocRun(judge_error=None, fill_error="503 from the takeaway endpoint") + judge_failed = _DocRun(judge_error="empty body twice", fill_error=None) + labels = label_runs([request_failed, judge_failed], passed=lambda r: False, judge_error=_judge_error_of) + assert labels.outcomes == (Outcome.failed(None), JUDGE) + + +def test_a_document_run_that_failed_a_fixed_check_is_a_failure_even_when_the_judge_errors() -> None: + run = _DocRun(judge_error="empty body twice", fill_error=None, checks_held=False) + labels = label_runs([run], passed=lambda r: False, judge_error=_judge_error_of) + assert labels.outcomes == (Outcome.failed(None),) + + +def test_an_unscored_run_gets_the_marker_only_when_the_harness_failed() -> None: + ctx = MagicMock() + mark_unscored_run(ctx, 0, "conv-1", JUDGE) + ctx.export_unscored.assert_not_called() + ctx.score.assert_called_once() + assert ctx.score.call_args.kwargs == { + "name": "eval_error", + "value": 1, + "data_type": "NUMERIC", + "comment": "judge_failed", + } + + ctx = MagicMock() + mark_unscored_run(ctx, 1, "conv-2", None) + ctx.export_unscored.assert_called_once_with(1, "conv-2") + ctx.score.assert_not_called() + + +# ── end to end through the visualization kind: labels, never a new verdict ─── + + +def _viz() -> dict: + return { + "id": "v1", + "type": "column_chart", + "query": { + "fields": {"m": {"using": "metric/revenue"}, "d": {"using": "label/date.quarter"}}, + "filter_by": {}, + }, + "metrics": ["m"], + "view_by": ["d"], + } + + +def _viz_turn() -> ChatResult: + return ChatResult.model_validate({"createdVisualizations": {"objects": [_viz()], "reasoning": ""}}) + + +def _question_turn() -> ChatResult: + return ChatResult.model_validate({"textResponse": "Which metric?"}) + + +def _evaluate(turns: list[Any], captured: dict[str, Any] | None = None, gate: str = "power") -> object: + client = MagicMock() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, len(turns) + 1)] + client.send_message.side_effect = turns + + def _capture(_submit: object, _identity: object, **kwargs: Any) -> None: + if captured is not None: + captured["write_scores"] = kwargs["write_scores"] + + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + patch.object(agentic_visualization, "generate_simulated_response", side_effect=RuntimeError("openai 401")), + patch.object(agentic_visualization, "submit_trace_scoring", _capture), + ): + return evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show revenue by quarter", + expected_outputs=[CreatedVisualization.model_validate(_viz())], + k=len(turns), + gate=gate, + langfuse=MagicMock() if captured is not None else None, + dataset_item_id="item-1" if captured is not None else None, + ) + + +def _silent_turn() -> ChatResult: + return ChatResult.model_validate({}) + + +def test_a_simulated_user_failure_keeps_the_verdict_and_is_labelled() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _viz_turn(), _viz_turn()]) + + exc = exc_info.value + message = str(exc) + lines = message.splitlines() + assert lines[1] == "Agentic Visualization Assertion Failed! (Critical Mode)" + # The gate counts the errored run as a run that did not pass: pass^3, 2/3. The note does + # not call that instability: the agent was not what failed it. + assert lines[2].startswith(gate_failure_note("power", 2, 3, runs_errored=1)) + assert "unstable" not in lines[2] + assert "run 1/3 ERROR (simulated_user_failed): visualization_created[expected true, actual false]" in lines[2] + assert (exc.runs_passed, exc.runs_effective) == (2, 3) + (record,) = exc.failed_runs + assert record["exit_reason"] == "simulated_user_failed" + assert record["outcome"] == {"status": "error", "reason": "simulated_user_failed"} + # Some runs were scored, so the item is not a harness error. + assert not hasattr(exc, "harness_error") + assert exc.shadow == {"verdict": "pass", "runs_scored": 2, "runs_total": 3} + + +def test_a_passing_item_records_its_shadow_verdict_too() -> None: + # pass@3 passes on the one scored run; the candidate rule, wanting ceil(3/2) scored runs, + # would not. Only a shadow recorded on a passing item shows that. + outcome = _evaluate([_question_turn(), _question_turn(), _viz_turn()], gate="any") + assert isinstance(outcome, AgenticEvalOutcome) + assert outcome.shadow == {"verdict": "error", "runs_scored": 1, "runs_total": 3} + assert [r["outcome"]["status"] for r in outcome.failed_runs] == ["error", "error"] + + +def test_an_item_whose_every_run_errored_is_labelled_and_still_fails() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _question_turn(), _question_turn()]) + + exc = exc_info.value + assert isinstance(exc, AgenticAssertionError) + assert exc.harness_error == {"status": "error", "reason": "simulated_user_failed"} + assert exc.shadow == {"verdict": "error", "runs_scored": 0, "runs_total": 3} + gate_line = str(exc).splitlines()[2] + assert gate_line.startswith( + f"Harness error on every run (simulated_user_failed). {gate_failure_note('power', 0, 3, runs_errored=3)}" + ) + assert exc.runs_passed == 0 + assert len(exc.failed_runs) == 3 + + +def test_the_head_says_no_run_was_scored_only_when_every_judge_failed() -> None: + labels = RunLabels((Outcome.error(JUDGE_FAILED), Outcome.error(JUDGE_FAILED))) + head = failure_message_head("power", labels, runs_passed=0, failed_runs=[], runs_ungraded=2) + assert head.startswith("Harness error, no run was scored (judge_failed). ") + + +def test_the_head_does_not_say_unscored_when_a_simulated_user_failed() -> None: + """A run whose simulated user failed was still checked; its failed checks follow the head.""" + labels = RunLabels((Outcome.error("simulated_user_failed"), Outcome.error(JUDGE_FAILED))) + head = failure_message_head("power", labels, runs_passed=0, failed_runs=[], runs_ungraded=1) + assert head.startswith("Harness error on every run (judge_failed). ") + assert "no run was scored" not in head + + +@pytest.mark.parametrize( + ("exc", "exit_value"), + [ + (TurnTimeoutError("agent exceeded the 60s turn budget"), "agent_timeout"), + (TurnIncompleteError("no final answer", reason="max iterations"), "turn_incomplete"), + (ChatError("SSE stream error"), "chat_error"), + ], +) +def test_a_chat_failure_is_a_plain_failure(exc: BaseException, exit_value: str) -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([exc, _viz_turn(), _viz_turn()]) + + raised = exc_info.value + assert (raised.runs_passed, raised.runs_effective) == (2, 3) + (record,) = raised.failed_runs + assert record["exit_reason"] == exit_value + assert record["outcome"] == {"status": "fail", "reason": exit_value} + assert f"run 1/3 FAIL (loop exit: {exit_value})" in str(raised) + assert "ERROR" not in str(raised) + assert not hasattr(raised, "harness_error") + assert raised.shadow["verdict"] == "fail" + + +def test_only_timeouts_is_a_failure_not_a_harness_error() -> None: + timeout = TurnTimeoutError("agent exceeded the 60s turn budget") + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([timeout, timeout, timeout]) + + assert not hasattr(exc_info.value, "harness_error") + assert str(exc_info.value).splitlines()[2].startswith(gate_failure_note("power", 0, 3)) + + +# ── Langfuse: an ERROR run gets an extra marker, every usual score stays ────── + + +class _ScoreCtx: + """A RunTraceContext stand-in that keeps each run's scores apart, by run index.""" + + def __init__(self) -> None: + self.run_metadata: dict[str, Any] = {} + self.scores: dict[int, list[tuple[str, Any, str | None]]] = defaultdict(list) + + def trace(self, _conversation_id: str) -> None: + return None + + @contextmanager + def observe(self, _trace: object, run_idx: int, **_kwargs: Any) -> Iterator[int]: + yield run_idx + + def score(self, trace_id: int, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[trace_id].append((name, value, comment)) + + def quality(self, trace_id: int, **_kwargs: Any) -> None: + self.scores[trace_id].append(("quality_score", None, None)) + + def names(self, run_idx: int) -> list[str]: + return [name for name, _, _ in self.scores[run_idx]] + + +def _scores(turns: list[Any]) -> _ScoreCtx: + captured: dict[str, Any] = {} + try: + _evaluate(turns, captured) + except AssertionError: + pass + ctx = _ScoreCtx() + captured["write_scores"](ctx) + return ctx + + +_USUAL_SCORES = {"gate_passed", "pass_at_k", "pass_power_k", "quality_score"} + + +def test_an_errored_run_gets_eval_error_beside_its_usual_scores() -> None: + ctx = _scores([_question_turn(), _viz_turn(), _viz_turn()]) + + assert ("eval_error", 1, "simulated_user_failed") in ctx.scores[0] + assert set(ctx.names(0)) >= _USUAL_SCORES + # The verdict scores are the ones the item was judged by: unchanged by the label. + assert dict((n, v) for n, v, _ in ctx.scores[0])["gate_passed"] is False + for run_idx in (1, 2): + assert "eval_error" not in ctx.names(run_idx) + assert set(ctx.names(run_idx)) >= _USUAL_SCORES + + +def test_a_chat_error_run_gets_no_marker() -> None: + ctx = _scores([ChatError("SSE stream error"), _viz_turn(), _viz_turn()]) + + for run_idx in range(3): + assert "eval_error" not in ctx.names(run_idx) + assert set(ctx.names(run_idx)) >= _USUAL_SCORES + + +# ── the message: the gate note and the failing runs, then the best run ─────── + +# The message of [simulated user failed, agent silent, pass] under pass^3, with the "Failing +# runs" part of its gate line left out. The best run's summary follows the gate line. +_MESSAGE_WITHOUT_FAILING_RUNS = ( + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + "Agentic Visualization Assertion Failed! (Critical Mode)\n" + "Gate pass^3 failed: 1/3 runs passed, 1 harness error(s) — unstable, not a clean failure.\n" + "------------------------------------------\n" + "Question:\n" + "Show revenue by quarter\n" + "------------------------------------------\n" + "Expected Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Actual Output:\n" + "{\n" + ' "id": "v1",\n' + ' "type": "column_chart",\n' + ' "query": {\n' + ' "fields": {\n' + ' "m": {\n' + ' "using": "metric/revenue"\n' + " },\n" + ' "d": {\n' + ' "using": "label/date.quarter"\n' + " }\n" + " },\n" + ' "filter_by": {},\n' + ' "sort_by": []\n' + " },\n" + ' "metrics": [\n' + ' "m"\n' + " ],\n" + ' "view_by": [\n' + ' "d"\n' + " ],\n" + ' "segment_by": [],\n' + ' "rows": [],\n' + ' "columns": [],\n' + ' "size_by": []\n' + "}\n" + "------------------------------------------\n" + "Strict Check Summary:\n" + " Visualization Created : True\n" + " Cross-Ref Valid : True\n" + " Metrics Correct : True\n" + " expected : ['metric/revenue']\n" + " actual : ['metric/revenue']\n" + " Dimensions Correct : True\n" + " expected : ['label/date.quarter']\n" + " actual : ['label/date.quarter']\n" + " Filters Correct : True\n" + " date : True\n" + " ranking : True\n" + " attribute : True\n" + " Viz Type Hard : True\n" + " Executed : False\n" + " Stated Value Matches : None (reported only)\n" + "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" +) + + +def test_the_agentic_message_is_the_gate_note_the_failing_runs_and_the_best_run() -> None: + with pytest.raises(VisualizationAssertionError) as exc_info: + _evaluate([_question_turn(), _silent_turn(), _viz_turn()]) + + message = str(exc_info.value) + gate_line = message.splitlines()[2] + head, failing = gate_line.split(" Failing runs: ", 1) + # A chartless run names only the missing chart, not every chart check that read False. + assert failing.startswith( + "run 1/3 ERROR (simulated_user_failed): visualization_created[expected true, actual false]; " + ) + assert failing.endswith( + "; run 2/3 FAIL (loop exit: agent_silent): visualization_created[expected true, actual false]." + ) + # The gate note names the errored run, and run 2 failed on its own beside a passing run 3, + # so the item is still unstable. + assert head == "Gate pass^3 failed: 1/3 runs passed, 1 harness error(s) — unstable, not a clean failure." + assert message.replace(gate_line, head, 1) == _MESSAGE_WITHOUT_FAILING_RUNS diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 629585358..6f28a767a 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -1091,6 +1091,73 @@ def forever(): assert sse_mod._is_retryable_exc(sse_mod.TurnTimeoutError("x")) is False +def _silent_after_one_line(now: list[float], silent_until: float): + """One event, then a read timeout that ends with the clock at ``silent_until``.""" + yield 'data: {"role":"assistant","content":{"type":"reasoning","text":"thinking"}}' + now[0] = silent_until + raise httpx.ReadTimeout("timed out") + + +def test_a_silent_agent_past_its_budget_is_a_turn_timeout(monkeypatch): + """The read timeout is lowered to the budget, so a silent agent ends in a read timeout, + not at the deadline check -- it is the same timeout all the same.""" + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + lines = sse_mod._until_deadline(_silent_after_one_line(now, 61.0), deadline=60.0, budget=60.0) + with pytest.raises(sse_mod.TurnTimeoutError, match="exceeded the 60s turn budget") as exc_info: + sse_mod.parse_sse_lines(lines) + assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) + + +def test_a_read_timeout_before_the_deadline_stays_a_stream_error(monkeypatch): + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + lines = sse_mod._until_deadline(_silent_after_one_line(now, 10.0), deadline=60.0, budget=60.0) + with pytest.raises(sse_mod.ChatError) as exc_info: + sse_mod.parse_sse_lines(lines) + assert not isinstance(exc_info.value, sse_mod.TurnTimeoutError) + assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) + + +def _headers_never_came(now: list[float], silent_until: float): + def handler(request): + now[0] = silent_until + raise httpx.ReadTimeout("timed out", request=request) + + return handler + + +def test_a_read_timeout_awaiting_the_headers_past_the_budget_is_a_turn_timeout(monkeypatch): + """A server that holds the headers back until the first event times out before the body.""" + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + client = _client_with_handler(_headers_never_came(now, 61.0), turn_timeout_s=60) + with pytest.raises(sse_mod.TurnTimeoutError, match="exceeded the 60s turn budget") as exc_info: + client.send_message("conv", "q") + assert isinstance(exc_info.value.__cause__, httpx.ReadTimeout) + + +def test_a_read_timeout_awaiting_the_headers_past_the_budget_records_the_turn(monkeypatch): + """The turn carries no partial result, but it is one of the slowest: the tail needs it.""" + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + client = _client_with_handler(_headers_never_came(now, 61.0), turn_timeout_s=60) + recorder = TraceRecorder() + with pytest.raises(sse_mod.TurnTimeoutError): + client.send_message("conv", "q") + assert recorder.stop() == [pytest.approx(61.0)] + + +def test_a_read_timeout_awaiting_the_headers_before_the_budget_is_left_as_it_is(monkeypatch): + now = [0.0] + monkeypatch.setattr(sse_mod.time, "monotonic", lambda: now[0]) + client = _client_with_handler(_headers_never_came(now, 10.0), turn_timeout_s=60) + recorder = TraceRecorder() + with pytest.raises(httpx.ReadTimeout): + client.send_message("conv", "q") + assert recorder.stop() == [] + + def test_no_turn_timeout_leaves_the_stream_untouched(): lines = ['data: {"role":"assistant","content":{"type":"text","text":"hi"}}'] assert list(sse_mod._until_deadline(iter(lines), deadline=None)) == lines