Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/gooddata-eval/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ experiment. The newest and most actively developed package in the repo.
| `core/reporting/` | console and JSON output rendering |
| `core/scoring.py`, `core/runner.py` | scoring and orchestration |
| `core/models.py` | `DatasetItem`, `ChatResult`, `ItemReport` and friends |
| `core/outcome.py` | `Outcome` (PASS / FAIL / ERROR): the one fail-closed rule that labels a failing run ERROR when the harness failed (simulated user, judge), plus the item label and the recorded-only shadow verdict. Labels decide no verdict; the agentic kinds reach it through `core/agentic/_outcome.py`'s `label_runs` |

**Depends on**: `gooddata-sdk`, `httpx`, `pydantic`, `orjson`, `rich`. The LLM-judge
evaluator is an optional extra (`llm-judge`, pulling `openai>=1.45,<2.0`); every `openai`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
from collections.abc import Callable, Sequence
from typing import Any

from gooddata_eval.core.agentic._failure_message import cap_failed_checks
from gooddata_eval.core.agentic._outcome import JudgeErrorOf, run_exit_reason, run_outcome
from gooddata_eval.core.models import build_tool_calls


Expand All @@ -33,6 +35,8 @@ def build_failed_runs(
*,
passed: Callable[[Any], bool],
detail: Callable[[Any], dict],
failed_checks: Callable[[Any], list[dict]] | None = None,
judge_error: JudgeErrorOf | None = None,
) -> list[dict]:
"""One record per run that did not pass, in the order the runs happened.

Expand All @@ -47,26 +51,37 @@ def build_failed_runs(
A run the judge could not grade is recorded like any other non-passing run, with its
``error`` set. That matches ``core/runner.py``, which also records ungraded runs; the
verdict-level accounting of ungraded runs stays with the caller's ``unscored_runs``.

``exit_reason`` is the run's ``LoopExit`` value, or null for a kind whose loop records
none, and ``outcome`` its ``core.outcome.Outcome`` record: ERROR when the harness failed
(its simulated user or its judge), FAIL otherwise. The label decides nothing; the run
counts as not passed either way; ``judge_error`` is passed on to ``run_outcome``. ``failed_checks`` is written only when the caller supplies it: a missing
key means the kind does not name its failing checks, while an empty list would claim the
run failed none. Its expected/actual values are capped in size; the run's ``detail``
keeps them whole.
"""
records: list[dict] = []
for run_index, run in enumerate(run_results, start=1):
if passed(run):
continue
tool_calls = list(getattr(run, "tool_call_events", None) or [])
reasoning_steps = list(getattr(run, "reasoning_steps", None) or [])
records.append(
{
"run_index": run_index,
"passed": False,
"error": getattr(run, "judge_error", None),
"detail": detail(run),
"conversation_id": getattr(run, "conversation_id", None),
"response_id": getattr(run, "response_id", None),
"reasoning_step_count": len(reasoning_steps),
"reasoning_steps": reasoning_steps,
"tool_call_count": len(tool_calls),
"tool_names": [getattr(e, "function_name", None) for e in tool_calls],
"tool_calls": build_tool_calls(tool_calls),
}
)
record: dict[str, Any] = {
"run_index": run_index,
"passed": False,
"error": getattr(run, "judge_error", None),
"detail": detail(run),
"conversation_id": getattr(run, "conversation_id", None),
"response_id": getattr(run, "response_id", None),
"reasoning_step_count": len(reasoning_steps),
"reasoning_steps": reasoning_steps,
"tool_call_count": len(tool_calls),
"tool_names": [getattr(e, "function_name", None) for e in tool_calls],
"tool_calls": build_tool_calls(tool_calls),
"exit_reason": run_exit_reason(run),
"outcome": run_outcome(run, passed=False, judge_error=judge_error).as_record(),
}
if failed_checks is not None:
record["failed_checks"] = cap_failed_checks(failed_checks(run))
records.append(record)
return records
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
# (C) 2026 GoodData Corporation
"""The head of an agentic kind's failure message: the gate result, then every failing run.

Each kind's assertion message describes its BEST run, which on a partial gate failure (some
of the K runs passed) is a passing run: every check reads True and the reason the item
failed is nowhere in the message. This module renders the part that is missing -- for each
failing run its index, why its loop stopped and the checks it failed, with expected/actual
where the kind knows them -- from the same ``failed_runs`` records the item reports, so the
message and the triage data cannot disagree about which runs failed or why.

One renderer for every kind on purpose: the readers of these messages (people and the
triage tooling) look for one shape, whatever the capability under test.
"""

from __future__ import annotations

import json
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from enum import Enum
from typing import Any

__all__ = ["FailedCheck", "cap_failed_checks", "failed_check_records", "failing_runs_note", "failure_head"]

# A value longer than this is cut: the message is one line per run, and the whole value is in
# the run's detail anyway.
_MAX_VALUE_CHARS = 160


class _Unset(Enum):
UNSET = "unset"


_UNSET = _Unset.UNSET


@dataclass(frozen=True)
class FailedCheck:
"""One check a run failed, with the expected/actual pair when the kind knows it.

``label`` narrows the check when one name covers several parts, e.g. the ``date`` part of
``filters_correct``.
"""

name: str
expected: Any = _UNSET
actual: Any = _UNSET
label: str | None = None

def as_record(self) -> dict[str, Any]:
"""The JSON-ready form written into a failing run's ``failed_checks``."""
record: dict[str, Any] = {"name": self.name}
if self.label is not None:
record["label"] = self.label
if self.expected is not _UNSET:
record["expected"] = self.expected
if self.actual is not _UNSET:
record["actual"] = self.actual
return record


def failed_check_records(checks: Iterable[FailedCheck]) -> list[dict[str, Any]]:
"""``checks`` as the records ``failed_runs[].failed_checks`` holds."""
return [check.as_record() for check in checks]


def _text(value: Any) -> str:
text = value if isinstance(value, str) else json.dumps(value, default=str, ensure_ascii=False, sort_keys=True)
return " ".join(text.split())


def _value(value: Any) -> str:
if value is None or value == [] or value == {} or value == "":
return "none"
text = _text(value)
return text if len(text) <= _MAX_VALUE_CHARS else text[: _MAX_VALUE_CHARS - 3] + "..."


def cap_failed_checks(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
"""``records`` with every expected/actual value cut to what the message shows of it.

A check's values can be whole filter or bucket structures, repeated for every failing run
of every item; the record keeps the same ``_MAX_VALUE_CHARS`` the message does. A value
that fits stays as it is, JSON and all; a longer one becomes its cut text. The run's
``detail`` holds the full values.
"""
capped: list[dict[str, Any]] = []
for record in records:
entry = dict(record)
for key in ("expected", "actual"):
if key in entry and len(_text(entry[key])) > _MAX_VALUE_CHARS:
entry[key] = _value(entry[key])
capped.append(entry)
return capped


def _render_check(record: Mapping[str, Any]) -> str:
name = str(record.get("name", "?"))
parts: list[str] = []
if "expected" in record:
parts.append(f"expected {_value(record['expected'])}")
if "actual" in record:
parts.append(f"actual {_value(record['actual'])}")
label = record.get("label")
if not parts and label is None:
return name
inner = ", ".join(parts)
if label is not None:
inner = f"{label}: {inner}" if inner else str(label)
return f"{name}[{inner}]"


def _render_run(record: Mapping[str, Any], runs_total: int) -> str:
outcome = record.get("outcome")
head = f"run {record.get('run_index', '?')}/{runs_total}"
exit_reason = record.get("exit_reason")
if isinstance(outcome, Mapping) and outcome.get("status") == "error":
# A harness error: the reason (simulated_user_failed, judge_failed) is the diagnosis.
# Its checks are not listed: they failed because the harness did, not the agent.
return f"{head} ERROR ({outcome.get('reason') or 'unknown'})"
head += " FAIL" + (f" (loop exit: {exit_reason})" if exit_reason else "")
checks = record.get("failed_checks")
if checks:
return f"{head}: {', '.join(_render_check(c) for c in checks)}"
if checks is not None:
return f"{head}: no failing check recorded"
return head


def failing_runs_note(failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str:
"""Every failing run on one line each, joined with ``; ``. Empty when no run failed."""
return "; ".join(_render_run(record, runs_total) for record in failed_runs)


def failure_head(gate_note: str, failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str:
"""``gate_note`` followed by the failing runs, ready to start a kind's assertion message.

Returns ``gate_note`` unchanged when there is no failing run to list, so a kind that
builds no ``failed_runs`` keeps its message exactly as before.
"""
note = failing_runs_note(failed_runs, runs_total)
return f"{gate_note} Failing runs: {note}." if note else gate_note
Loading
Loading