Repository navigation
feat(gooddata-eval): label harness errors, keep every verdict #1960
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| # (C) 2026 GoodData Corporation | ||
| """The head of an agentic kind's failure message: the gate result, then every failing run. | ||
|
|
||
| Each kind's assertion message describes its BEST run, which on a partial gate failure (some | ||
| of the K runs passed) is a passing run: every check reads True and the reason the item | ||
| failed is nowhere in the message. This module renders the part that is missing -- for each | ||
| failing run its index, why its loop stopped and the checks it failed, with expected/actual | ||
| where the kind knows them -- from the same ``failed_runs`` records the item reports, so the | ||
| message and the triage data cannot disagree about which runs failed or why. | ||
|
|
||
| One renderer for every kind that labels its runs, on purpose: the readers of these messages | ||
| (people and the triage tooling) look for one shape, whatever the capability under test. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import json | ||
| from collections.abc import Iterable, Mapping, Sequence | ||
| from dataclasses import dataclass | ||
| from enum import Enum | ||
| from typing import Any | ||
|
|
||
| from gooddata_eval.core.outcome import UNKNOWN_REASON, OutcomeStatus | ||
|
|
||
| __all__ = [ | ||
| "FailedCheck", | ||
| "cap_failed_checks", | ||
| "failed_check_records", | ||
| "failed_checks_from", | ||
| "failing_runs_note", | ||
| "failure_head", | ||
| "judge_failed_checks", | ||
| ] | ||
|
|
||
| # A value longer than this is cut: the message is one line per run, and the whole value is in | ||
| # the run's detail anyway. | ||
| _MAX_VALUE_CHARS = 160 | ||
|
|
||
| # The failing runs together are cut past this, at a run boundary: K runs of a kind with a dozen | ||
| # checks would otherwise put tens of thousands of characters into one assertion message. Every | ||
| # run is in ``failed_runs`` whatever the message shows. | ||
| _MAX_FAILING_RUNS_CHARS = 2000 | ||
|
|
||
|
|
||
| class _Unset(Enum): | ||
| UNSET = "unset" | ||
|
|
||
|
|
||
| _UNSET = _Unset.UNSET | ||
|
|
||
|
|
||
| @dataclass(frozen=True) | ||
| class FailedCheck: | ||
| """One check a run failed, with the expected/actual pair when the kind knows it. | ||
|
|
||
| ``part`` narrows the check when one name covers several parts, e.g. the ``date`` part of | ||
| ``filters_correct``. | ||
| """ | ||
|
|
||
| name: str | ||
| expected: Any = _UNSET | ||
| actual: Any = _UNSET | ||
| part: str | None = None | ||
|
|
||
| def as_record(self) -> dict[str, Any]: | ||
| """The JSON-ready form written into a failing run's ``failed_checks``.""" | ||
| record: dict[str, Any] = {"name": self.name} | ||
| if self.part is not None: | ||
| record["part"] = self.part | ||
| if self.expected is not _UNSET: | ||
| record["expected"] = self.expected | ||
| if self.actual is not _UNSET: | ||
| record["actual"] = self.actual | ||
| return record | ||
|
|
||
|
|
||
| def failed_check_records(checks: Iterable[FailedCheck]) -> list[dict[str, Any]]: | ||
| """``checks`` as the records ``failed_runs[].failed_checks`` holds.""" | ||
| return [check.as_record() for check in checks] | ||
|
|
||
|
|
||
| def failed_checks_from(checks: Mapping[str, bool], *, prerequisite: str | None = None) -> list[dict[str, Any]]: | ||
| """The records of the checks in ``checks`` that read False, by name, in their order. | ||
|
|
||
| ``prerequisite`` names the check every other one depends on, e.g. ``alert_created``. When it | ||
| failed, the others read False only because there was nothing to check, so it alone is named. | ||
| """ | ||
| if prerequisite is not None and checks.get(prerequisite) is False: | ||
| return failed_check_records([FailedCheck(prerequisite)]) | ||
| return failed_check_records(FailedCheck(name) for name, ok in checks.items() if not ok) | ||
|
|
||
|
|
||
| def judge_failed_checks(name: str, *, passed: bool, judge_error: str | None, reasoning: Any) -> list[dict[str, Any]]: | ||
| """The judge's verdict ``name`` on a run it failed, with its reasoning as the actual value. | ||
|
|
||
| Empty when the judge passed the run, or could not grade it: a run with no verdict failed no | ||
| judge check. | ||
| """ | ||
| if judge_error is not None or passed: | ||
| return [] | ||
| return failed_check_records([FailedCheck(name, part="judge", actual=reasoning)]) | ||
|
|
||
|
|
||
| def _text(value: Any) -> str: | ||
| text = value if isinstance(value, str) else json.dumps(value, default=str, ensure_ascii=False, sort_keys=True) | ||
| return " ".join(text.split()) | ||
|
|
||
|
|
||
| def _value(value: Any) -> str: | ||
| if value is None or value == [] or value == {} or value == "": | ||
| return "none" | ||
| text = _text(value) | ||
| return text if len(text) <= _MAX_VALUE_CHARS else text[: _MAX_VALUE_CHARS - 3] + "..." | ||
|
|
||
|
|
||
| def cap_failed_checks(records: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: | ||
| """``records`` with every expected/actual value cut to what the message shows of it. | ||
|
|
||
| A check's values can be whole filter or bucket structures, repeated for every failing run | ||
| of every item; the record keeps the same ``_MAX_VALUE_CHARS`` the message does. A value | ||
| that fits stays as it is, JSON and all; a longer one becomes its cut text. The run's | ||
| ``detail`` holds the full values. | ||
| """ | ||
| capped: list[dict[str, Any]] = [] | ||
| for record in records: | ||
| entry = dict(record) | ||
| for key in ("expected", "actual"): | ||
| if key in entry and len(_text(entry[key])) > _MAX_VALUE_CHARS: | ||
| entry[key] = _value(entry[key]) | ||
| capped.append(entry) | ||
| return capped | ||
|
|
||
|
|
||
| def _render_check(record: Mapping[str, Any]) -> str: | ||
| name = str(record.get("name", "?")) | ||
| parts: list[str] = [] | ||
| if "expected" in record: | ||
| parts.append(f"expected {_value(record['expected'])}") | ||
| if "actual" in record: | ||
| parts.append(f"actual {_value(record['actual'])}") | ||
| part = record.get("part") | ||
| if not parts and part is None: | ||
| return name | ||
| inner = ", ".join(parts) | ||
| if part is not None: | ||
| inner = f"{part}: {inner}" if inner else str(part) | ||
| return f"{name}[{inner}]" | ||
|
|
||
|
|
||
| def _render_run(record: Mapping[str, Any], runs_total: int) -> str: | ||
| outcome = record.get("outcome") | ||
| head = f"run {record.get('run_index', '?')}/{runs_total}" | ||
| exit_reason = record.get("exit_reason") | ||
| errored = isinstance(outcome, Mapping) and outcome.get("status") == OutcomeStatus.ERROR.value | ||
| if errored: | ||
| # A harness error: the reason (simulated_user_failed, judge_failed) is the diagnosis. | ||
| # Its checks are still listed: the simulated user is also asked after an answer that | ||
| # was already final, and what that answer missed is the only way to tell. | ||
| head += f" ERROR ({outcome.get('reason') or UNKNOWN_REASON})" | ||
| else: | ||
| head += " FAIL" + (f" (loop exit: {exit_reason})" if exit_reason else "") | ||
| checks = record.get("failed_checks") | ||
| if checks: | ||
| return f"{head}: {', '.join(_render_check(c) for c in checks)}" | ||
| if checks is not None and not errored: | ||
| return f"{head}: no failing check recorded" | ||
| return head | ||
|
|
||
|
|
||
| def failing_runs_note(failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: | ||
| """Every failing run, joined with ``; ``. Empty when no run failed. | ||
|
|
||
| Past ``_MAX_FAILING_RUNS_CHARS`` the runs left are counted instead of rendered. The first | ||
| run is always shown, cut to that length if it alone is longer. | ||
| """ | ||
| shown: list[str] = [] | ||
| length = 0 | ||
| for index, record in enumerate(failed_runs): | ||
| text = _render_run(record, runs_total) | ||
| if shown and length + len("; ") + len(text) > _MAX_FAILING_RUNS_CHARS: | ||
| more = len(failed_runs) - index | ||
| shown.append(f"{more} more failing run(s) in failed_runs") | ||
| break | ||
| if not shown and len(text) > _MAX_FAILING_RUNS_CHARS: | ||
| text = text[: _MAX_FAILING_RUNS_CHARS - 3] + "..." | ||
| shown.append(text) | ||
| length += len(text) + (len("; ") if len(shown) > 1 else 0) | ||
| return "; ".join(shown) | ||
|
|
||
|
|
||
| def failure_head(gate_note: str, failed_runs: Sequence[Mapping[str, Any]], runs_total: int) -> str: | ||
| """``gate_note`` followed by the failing runs, ready to start a kind's assertion message. | ||
|
|
||
| Returns ``gate_note`` unchanged when there is no failing run to list, which is the whole | ||
| head of a kind that builds no ``failed_runs``. | ||
| """ | ||
| note = failing_runs_note(failed_runs, runs_total) | ||
| return f"{gate_note} Failing runs: {note}." if note else gate_note |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -29,20 +29,27 @@ def gate_label(gate: str | None, k: int) -> str: | |
| return f"pass^{k}" if normalize_gate(gate) == "power" else f"pass@{k}" | ||
|
|
||
|
|
||
| def gate_failure_note(gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0) -> str: | ||
| def gate_failure_note( | ||
| gate: str | None, runs_passed: int, runs_total: int, runs_ungraded: int = 0, *, runs_errored: int = 0 | ||
| ) -> str: | ||
| """Gate and how many runs met it, for the assertion message. | ||
|
|
||
| Needed because the message body describes the BEST run, which under pass^K can be a run | ||
| that passed — so the reported detail on its own looks like a pass. | ||
|
|
||
| An ungraded run counts in runs_total but can never count in runs_passed, so the remainder | ||
| is not evidence of instability — saying so would blame the agent for a judge outage. | ||
| is not evidence of instability — saying so would blame the agent for a judge outage. A run | ||
| the harness failed (``runs_errored``, labelled ERROR in ``core/outcome.py``) is no evidence | ||
| of instability either, so it is named rather than called one. A pass^K item with a passing | ||
| run and a run the agent itself failed is unstable all the same, whatever else errored. | ||
| """ | ||
| label = gate_label(gate, runs_total) | ||
| note = f"Gate {label} failed: {runs_passed}/{runs_total} runs passed" | ||
| if runs_ungraded: | ||
| return f"{note}, {runs_ungraded} ungraded." | ||
| if normalize_gate(gate) == "power" and runs_passed: | ||
| if runs_errored: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. With any ERROR run present, this returns before the pass^K "unstable" branch. pass^3 runs Suggestion: name the errors and still add the instability note when |
||
| note = f"{note}, {runs_errored} harness error(s)" | ||
| if normalize_gate(gate) == "power" and runs_passed and runs_passed + runs_errored < runs_total: | ||
| return f"{note} — unstable, not a clean failure." | ||
| return f"{note}." | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Report harness-error counts when some runs are ungraded.
When
runs_ungradedis nonzero, the early return at lines 48-49 drops theruns_erroredsuffix. A run that failed in the simulated user is labelled ERROR, and it is not ungraded. Such a run is hidden when another run is ungraded. For example, in a pass^3 item with one passing run, one ungraded judge-error run that also failed other checks, and one simulated-user harness error, the note reads1/3 runs passed, 1 ungraded.The harness error is not shown.Build both count suffixes before the early return. Keep the instability message suppressed when ungraded runs are present.
Suggested fix