Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/gooddata-eval/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ experiment. The newest and most actively developed package in the repo.
| `core/reporting/` | console and JSON output rendering |
| `core/scoring.py`, `core/runner.py` | scoring and orchestration |
| `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends |
| `core/outcome.py` | `Outcome` (PASS / FAIL / ERROR): the one fail-closed rule that labels a failing run ERROR when the harness failed (simulated user, judge), plus the item label and the recorded-only shadow verdict. Labels decide no verdict; the agentic kinds reach it through `core/agentic/_outcome.py`'s `label_runs` |

**Depends on**: `gooddata-sdk`, `httpx`, `pydantic`, `orjson`, `rich`. The LLM-judge
evaluator is an optional extra (`llm-judge`, pulling `openai>=1.45,<2.0`); every `openai`
Expand Down
13 changes: 13 additions & 0 deletions packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,16 @@ def _apply_run_counts(item_report: ItemReport, source: Any) -> None:
item_report.runs_ungraded = unscored


def _apply_labels(item_report: ItemReport, source: Any) -> None:
"""Copy the item's harness label and shadow verdict off an outcome or a failure, if set."""
harness_error = getattr(source, "harness_error", None)
if isinstance(harness_error, dict):
item_report.harness_error = harness_error
shadow = getattr(source, "shadow", None)
if isinstance(shadow, dict):
item_report.shadow = shadow


def _apply_failed_runs(item_report: ItemReport, source: Any) -> None:
"""Copy the per-run failure records off an outcome or a failure, if the kind built any.

Expand Down Expand Up @@ -555,6 +565,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
_apply_best_run_latency(item_report, outcome)
_apply_run_counts(item_report, outcome)
_apply_failed_runs(item_report, outcome)
_apply_labels(item_report, outcome)
_apply_run_latencies(item_report, outcome)
except AssertionError as exc:
item_report.gate_passed = False if gated else None
Expand All @@ -567,6 +578,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
_apply_best_run_latency(item_report, exc)
_apply_run_counts(item_report, exc)
_apply_failed_runs(item_report, exc)
_apply_labels(item_report, exc)
_apply_run_latencies(item_report, exc)
# Read off the counts, not off the gate: pass^K fails items where runs did pass,
# and reporting those as pass_at_k False would contradict the Langfuse score of
Expand All @@ -588,6 +600,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
_apply_timings(item_report, getattr(exc, "timings", None))
_apply_run_counts(item_report, exc)
_apply_failed_runs(item_report, exc)
_apply_labels(item_report, exc)
_apply_run_latencies(item_report, exc)
# Every run ungraded, by definition of this error -- said explicitly rather than
# left to _apply_run_counts, whose source is detail["unscored_runs"] and which a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,28 @@
from collections.abc import Callable, Sequence
from typing import Any

from gooddata_eval.core.agentic._failure_message import cap_failed_checks
from gooddata_eval.core.agentic._outcome import RunLabels, run_exit_reason, run_outcome
from gooddata_eval.core.models import build_tool_calls
from gooddata_eval.core.outcome import OutcomeStatus


def build_failed_runs(
run_results: Sequence[Any],
*,
passed: Callable[[Any], bool],
detail: Callable[[Any], dict],
labels: RunLabels | None = None,
passed: Callable[[Any], bool] | None = None,
failed_checks: Callable[[Any], list[dict]] | None = None,
) -> list[dict]:
"""One record per run that did not pass, in the order the runs happened.

A kind that labels its runs passes its ``labels`` (from ``label_runs``, in run order):
they say which runs did not pass and are the ``outcome`` of each record, so the records,
the failure message and the Langfuse ``eval_error`` marker read one set of labels. A kind
that does not label its runs passes ``passed`` instead, and each record's ``outcome`` is
worked out here from the run alone. Exactly one of the two is given.

``passed`` and ``detail`` are supplied by the caller because neither is uniform across
kinds: guardrail reads ``run.passed`` while visualization reads
``run.eval_result.strict_pass``, and each kind's diagnostic dict is its own. Everything
Expand All @@ -45,28 +56,51 @@ def build_failed_runs(
``ToolCallEvent``s, so a run that has the list must hold real events in it.

A run the judge could not grade is recorded like any other non-passing run, with its
``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the
verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``.
``error`` set to the run's ``judge_error``. That matches ``core/runner.py``, which also
records ungraded runs; the verdict-level accounting of ungraded runs stays with the
caller's ``unscored_runs``.

``exit_reason`` is the run's ``LoopExit`` value, or null for a kind whose loop records
none, and ``outcome`` its ``core.outcome.Outcome`` record: ERROR when the harness failed
(its simulated user, or its judge when the judge's verdict was the one still open), FAIL
otherwise. ``error`` and ``outcome`` can differ: a run that failed a fixed check and whose
judge also broke has ``error`` set and is a FAIL. The label decides nothing; the run counts
as not passed either way.

``failed_checks`` is written only when the caller supplies it: a missing key means the
kind does not name its failing checks, while an empty list would claim the run failed
none. Its expected/actual values are capped in size; the run's ``detail`` keeps them whole.
"""
if (labels is None) == (passed is None):
raise ValueError("build_failed_runs takes exactly one of labels and passed")
if labels is not None and len(labels.outcomes) != len(run_results):
raise ValueError(f"{len(labels.outcomes)} labels for {len(run_results)} runs")
records: list[dict] = []
for run_index, run in enumerate(run_results, start=1):
if passed(run):
if labels is not None:
outcome = labels.outcomes[run_index - 1]
else:
outcome = run_outcome(run, passed=passed is not None and passed(run))
if outcome.status is OutcomeStatus.PASS:
continue
tool_calls = list(getattr(run, "tool_call_events", None) or [])
reasoning_steps = list(getattr(run, "reasoning_steps", None) or [])
records.append(
{
"run_index": run_index,
"passed": False,
"error": getattr(run, "judge_error", None),
"detail": detail(run),
"conversation_id": getattr(run, "conversation_id", None),
"response_id": getattr(run, "response_id", None),
"reasoning_step_count": len(reasoning_steps),
"reasoning_steps": reasoning_steps,
"tool_call_count": len(tool_calls),
"tool_names": [getattr(e, "function_name", None) for e in tool_calls],
"tool_calls": build_tool_calls(tool_calls),
}
)
record: dict[str, Any] = {
"run_index": run_index,
"passed": False,
"error": getattr(run, "judge_error", None),
"detail": detail(run),
"conversation_id": getattr(run, "conversation_id", None),
"response_id": getattr(run, "response_id", None),
"reasoning_step_count": len(reasoning_steps),
"reasoning_steps": reasoning_steps,
"tool_call_count": len(tool_calls),
"tool_names": [getattr(e, "function_name", None) for e in tool_calls],
"tool_calls": build_tool_calls(tool_calls),
"exit_reason": run_exit_reason(run),
"outcome": outcome.as_record(),
}
if failed_checks is not None:
record["failed_checks"] = cap_failed_checks(failed_checks(run))
records.append(record)
return records
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
# (C) 2026 GoodData Corporation
"""The head of an agentic kind's failure message: the gate result, then every failing run.

Each kind's assertion message describes its BEST run, which on a partial gate failure (some
of the K runs passed) is a passing run: every check reads True and the reason the item
failed is nowhere in the message. This module renders the part that is missing -- for each
failing run its index, why its loop stopped and the checks it failed, with expected/actual
where the kind knows them -- from the same ``failed_runs`` records the item reports, so the
message and the triage data cannot disagree about which runs failed or why.

One renderer for every kind that labels its runs, on purpose: the readers of these messages
(people and the triage tooling) look for one shape, whatever the capability under test.
"""

from __future__ import annotations

import json
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from enum import Enum
from typing import Any

from gooddata_eval.core.outcome import UNKNOWN_REASON, OutcomeStatus

__all__ = [
"FailedCheck",
"cap_failed_checks",
"failed_check_records",
"failed_checks_from",
"failing_runs_note",
"failure_head",
"judge_failed_checks",
]

# A value longer than this is cut: the message is one line per run, and the whole value is in
# the run's detail anyway.
_MAX_VALUE_CHARS = 160

# The failing runs together are cut past this, at a run boundary: K runs of a kind with a dozen
# checks would otherwise put tens of thousands of characters into one assertion message. Every
# run is in ``failed_runs`` whatever the message shows.
_MAX_FAILING_RUNS_CHARS = 2000


class _Unset(Enum):
UNSET = "unset"


_UNSET = _Unset.UNSET


@dataclass(frozen=True)
class FailedCheck:
"""One check a run failed, with the expected/actual pair when the kind knows it.

``part`` narrows the check when one name covers several parts, e.g. the ``date`` part of
``filters_correct``.
"""

name: str
expected: Any = _UNSET
actual: Any = _UNSET
part: str | None = None

def as_record(self) -> dict[str, Any]:
"""The JSON-ready form written into a failing run's ``failed_checks``."""
record: dict[str, Any] = {"name": self.name}
if self.part is not None:
record["part"] = self.part
if self.expected is not _UNSET:
record["expected"] = self.expected
if self.actual is not _UNSET:
record["actual"] = self.actual
return record


def failed_check_records(checks: Iterable[FailedCheck]) -> list[dict[str, Any]]:
"""``checks`` as the records ``failed_runs[].failed_checks`` holds."""
return [check.as_record() for check in checks]


def failed_checks_from(checks: Mapping[str, bool], *, prerequisite: str | None = None) -> list[dict[str, Any]]:
"""The records of the checks in ``checks`` that read False, by name, in their order.

``prerequisite`` names the check every other one depends on, e.g. ``alert_created``. When it
failed, the others read False only because there was nothing to check, so it alone is named.
"""
if prerequisite is not None and checks.get(prerequisite) is False:
return failed_check_records([FailedCheck(prerequisite)])
return failed_check_records(FailedCheck(name) for name, ok in checks.items() if not ok)


def judge_failed_checks(name: str, *, passed: bool, judge_error: str | None, reasoning: Any) -> list[dict[str, Any]]:
"""The judge's verdict ``name`` on a run it failed, with its reasoning as the actual value.

Empty when the judge passed the run, or could not grade it: a run with no verdict failed no
judge check.
"""
if judge_error is not None or passed:
return []
return failed_check_records([FailedCheck(name, part="judge", actual=reasoning)])


def _text(value: Any) -> str:
text = value if isinstance(value, str) else json.dumps(value, default=str, ensure_ascii=False, sort_keys=True)
return " ".join(text.split())


def _value(value: Any) -> str:
if value is None or value == [] or value == {} or value == "":
return "none"
text = _text(value)
return text if len(text) <= _MAX_VALUE_CHARS else text[: _MAX_VALUE_CHARS - 3] + "..."


def cap_failed_checks(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
"""``records`` with every expected/actual value cut to what the message shows of it.

A check's values can be whole filter or bucket structures, repeated for every failing run
of every item; the record keeps the same ``_MAX_VALUE_CHARS`` the message does. A value
that fits stays as it is, JSON and all; a longer one becomes its cut text. The run's
``detail`` holds the full values.
"""
capped: list[dict[str, Any]] = []
for record in records:
entry = dict(record)
for key in ("expected", "actual"):
if key in entry and len(_text(entry[key])) > _MAX_VALUE_CHARS:
entry[key] = _value(entry[key])
capped.append(entry)
return capped


def _render_check(record: Mapping[str, Any]) -> str:
name = str(record.get("name", "?"))
parts: list[str] = []
if "expected" in record:
parts.append(f"expected {_value(record['expected'])}")
if "actual" in record:
parts.append(f"actual {_value(record['actual'])}")
part = record.get("part")
if not parts and part is None:
return name
inner = ", ".join(parts)
if part is not None:
inner = f"{part}: {inner}" if inner else str(part)
return f"{name}[{inner}]"


def _render_run(record: Mapping[str, Any], runs_total: int) -> str:
outcome = record.get("outcome")
head = f"run {record.get('run_index', '?')}/{runs_total}"
exit_reason = record.get("exit_reason")
errored = isinstance(outcome, Mapping) and outcome.get("status") == OutcomeStatus.ERROR.value
if errored:
# A harness error: the reason (simulated_user_failed, judge_failed) is the diagnosis.
# Its checks are still listed: the simulated user is also asked after an answer that
# was already final, and what that answer missed is the only way to tell.
head += f" ERROR ({outcome.get('reason') or UNKNOWN_REASON})"
else:
head += " FAIL" + (f" (loop exit: {exit_reason})" if exit_reason else "")
checks = record.get("failed_checks")
if checks:
return f"{head}: {', '.join(_render_check(c) for c in checks)}"
if checks is not None and not errored:
return f"{head}: no failing check recorded"
return head


def failing_runs_note(failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str:
"""Every failing run, joined with ``; ``. Empty when no run failed.

Past ``_MAX_FAILING_RUNS_CHARS`` the runs left are counted instead of rendered. The first
run is always shown, cut to that length if it alone is longer.
"""
shown: list[str] = []
length = 0
for index, record in enumerate(failed_runs):
text = _render_run(record, runs_total)
if shown and length + len("; ") + len(text) > _MAX_FAILING_RUNS_CHARS:
more = len(failed_runs) - index
shown.append(f"{more} more failing run(s) in failed_runs")
break
if not shown and len(text) > _MAX_FAILING_RUNS_CHARS:
text = text[: _MAX_FAILING_RUNS_CHARS - 3] + "..."
shown.append(text)
length += len(text) + (len("; ") if len(shown) > 1 else 0)
return "; ".join(shown)


def failure_head(gate_note: str, failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str:
"""``gate_note`` followed by the failing runs, ready to start a kind's assertion message.

Returns ``gate_note`` unchanged when there is no failing run to list, which is the whole
head of a kind that builds no ``failed_runs``.
"""
note = failing_runs_note(failed_runs, runs_total)
return f"{gate_note} Failing runs: {note}." if note else gate_note
13 changes: 10 additions & 3 deletions packages/gooddata-eval/src/gooddata_eval/core/agentic/_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,27 @@ def gate_label(gate: str | None, k: int) -> str:
return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}"


def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0) -> str:
def gate_failure_note(
gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0, *, runs_errored: int = 0
) -> str:
"""Gate and how many runs met it, for the assertion message.

Needed because the message body describes the BEST run, which under pass^K can be a run
that passed — so the reported detail on its own looks like a pass.

An ungraded run counts in runs_total but can never count in runs_passed, so the remainder
is not evidence of instability — saying so would blame the agent for a judge outage.
is not evidence of instability — saying so would blame the agent for a judge outage. A run
the harness failed (``runs_errored``, labelled ERROR in ``core/outcome.py``) is no evidence
of instability either, so it is named rather than called one. A pass^K item with a passing
run and a run the agent itself failed is unstable all the same, whatever else errored.
"""
label = gate_label(gate, runs_total)
note = f"Gate {label} failed: {runs_passed}/{runs_total} runs passed"
if runs_ungraded:
return f"{note}, {runs_ungraded} ungraded."
if normalize_gate(gate) == "power" and runs_passed:
if runs_errored:
note = f"{note}, {runs_errored} harness error(s)"
if normalize_gate(gate) == "power" and runs_passed and runs_passed + runs_errored < runs_total:
return f"{note} — unstable, not a clean failure."
return f"{note}."

Expand Down
Loading
Loading