From e64a6645ba2120c0b65d8c735709a9f3b6a0edec Mon Sep 17 00:00:00 2001 From: huyenthanh09 Date: Thu, 19 Nov 2026 15:13:33 +0700 Subject: [PATCH] feat(gooddata-eval): opt-in search-tool checks beyond the tool call The search_tool items pass as soon as search_objects is called, so an item cannot tell whether the agent found the right objects, showed the search widget, kept to the search skill or answered honestly. Ten of the twelve items name objects the fixture lacks and still pass. An item can now opt into further checks beside tool_call. Each key adds one score and gates the run; an item without them scores as before: - expected_objects: every ref is returned by a search_objects result or a searchResults part. A search the agent runs for itself shows no widget, so its tool result counts. attribute, label and date_attribute name the same object - expect_widget: a searchResults part is present, or absent - required_tools / forbidden_tools: the tools of the scored turn - search_required: false lifts the tool_selection gate for routing items; their search verdicts move to the detail's diagnostics - answer_rubric: the LLM judge grades the last reply's text; a turn without text fails without a judge request - expected_limit: any search_objects call asked for this limit - no_raw_ids: no UUID, and no type/id with an underscore, dot or digit, outside a {type/id} chip, which the UI renders as the object's title - setup_turns: sent first in the same conversation, not scored; they must only read the workspace, as agentic_search runs in parallel - report_only: the checks are published as diagnostics and do not gate Unknown keys and malformed values raise before any request. A judge without a verdict labels the run ERROR when nothing else failed it, and its score is not published. A setup turn that fails is the agent's FAIL with the chat exit reason, as a failed chat turn is for every kind. The evaluators take the whole expected output as expected_output. The CLI passes it whole unless it holds only search_objects arguments, which still go in expected_tool_call. tool_correctness now accepts any search_objects call of the turn and compares compatible_with_afm loosely: the agent often looks a metric up first and asks for compatible attributes after. It is published only when the item names arguments. Re-scored on 36 saved runs of the 12 current items, no tool_correctness value changes. SEARCH_CONDITIONAL_SCORES lists the published check names. jira: QA-29908 risk: low Co-Authored-By: Claude Opus 5.5 --- .../src/gooddata_eval/cli/agentic_runner.py | 12 +- .../core/agentic/_search_expectation.py | 301 ++++++++ .../gooddata_eval/core/agentic/search_tool.py | 444 +++++++++--- .../tests/test_agentic_search_tool_checks.py | 676 ++++++++++++++++++ 4 files changed, 1341 insertions(+), 92 deletions(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/_search_expectation.py create mode 100644 packages/gooddata-eval/tests/test_agentic_search_tool_checks.py diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index e596c6128..46b458ff2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -11,6 +11,7 @@ from gooddata_eval.core.agentic._gate import DEFAULT_GATE, EvalGate, normalize_gate from gooddata_eval.core.agentic._langfuse import make_langfuse_client +from gooddata_eval.core.agentic._search_expectation import SEARCH_ARGUMENT_KEYS from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection @@ -253,15 +254,18 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_search": - eo_dict = eo if isinstance(eo, dict) else {} - tool_call = eo_dict.get("tool_call", {}) - expected_args = tool_call.get("function_arguments", eo_dict) + if not isinstance(eo, dict): + raise ValueError(f"agentic_search expected_output must be an object, got {eo!r}") + eo_dict = eo + # Anything but bare search_objects arguments is parsed as an item, so an unknown key fails it. + is_item_shape = "tool_call" in eo_dict or not eo_dict.keys() <= SEARCH_ARGUMENT_KEYS return evaluate_agentic_search_tool( host=host, token=token, workspace_id=workspace_id, question=item.question, - expected_tool_call=expected_args, + expected_tool_call=None if is_item_shape else eo_dict, + expected_output=eo_dict if is_item_shape else None, k=k, gate=gate, agent_id=agent_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_search_expectation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_search_expectation.py new file mode 100644 index 000000000..68086e356 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_search_expectation.py @@ -0,0 +1,301 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""The search-tool item: the expected ``search_objects`` arguments plus the checks an item opts into. + +An item that carries only ``tool_call``, or reaches the runner as the bare arguments, is scored on +``tool_selection`` (gating) and ``tool_correctness`` (informing). Each other key adds one check, and a +key the module does not know is rejected, so a misspelt check cannot read as a pass. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.models import ChatResult, ToolCallEvent + +SEARCH_TOOL = "search_objects" + +# The keys of an item's expected output beside ``tool_call``. Public so a caller holding a raw item can +# tell the options shape from bare arguments. +SEARCH_OPTION_KEYS = frozenset( + { + "expected_objects", + "expect_widget", + "required_tools", + "forbidden_tools", + "search_required", + "answer_rubric", + "expected_limit", + "no_raw_ids", + "setup_turns", + "report_only", + } +) +_KNOWN_KEYS = SEARCH_OPTION_KEYS | {"tool_call"} +# The search_objects parameters, and emit_widget, the old name of user_requested_search. An expected output with +# any other key is read as an item, so a misspelt check key fails it instead of passing as an argument. +SEARCH_ARGUMENT_KEYS = frozenset( + {"keywords", "object_types", "limit", "compatible_with_afm", "kda_eligible", "user_requested_search", "emit_widget"} +) + +_OBJECT_TYPES = frozenset( + { + "metric", + "fact", + "attribute", + "label", + "date_attribute", + "computed_attribute", + "dataset", + "date", + "parameter", + "visualization", + "dashboard", + "document", + "report", + } +) +# The agent's tools take an attribute by either of its ids, and search returns a date attribute with +# ``type`` attribute and ``final_type`` date_attribute, so these spellings name one object. +_ATTRIBUTE_TYPES = frozenset({"attribute", "label", "date_attribute"}) + +# A ``{type/id}`` chip is the UI's own syntax for a named object and renders as its title, also inside +# inline code. An id outside a chip reaches the user as raw text. +_CHIP = re.compile(r"\{[a-z_]+/[^{}\s]+\}") +_UUID = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE) +_TYPED_ID = re.compile(rf"\b(?:{'|'.join(sorted(_OBJECT_TYPES))})/([A-Za-z0-9_](?:[A-Za-z0-9_.-]*[A-Za-z0-9_])?)") +# Prose pairs type words with a slash ("date/time", "label/value pairs"). Only an id with an underscore, +# a dot or a digit, as the workspace ids have, counts. A one-word id such as metric/revenue is not caught. +_ID_MARK = re.compile(r"[_.0-9]") + + +@dataclass(frozen=True) +class SearchExpectation: + """What one item expects. Every field but ``tool_args`` is an opt-in check. + + ``setup_turns`` must only read the workspace (search, describe, an unsaved chart): ``agentic_search`` runs in + the CLI's parallel phase, beside other items of the same workspace. + """ + + tool_args: dict[str, Any] = field(default_factory=dict) + expected_objects: tuple[str, ...] = () + expect_widget: bool | None = None + required_tools: tuple[str, ...] = () + forbidden_tools: tuple[str, ...] = () + search_required: bool = True + answer_rubric: tuple[str, ...] = () + expected_limit: int | None = None + no_raw_ids: bool = False + setup_turns: tuple[str, ...] = () + report_only: bool = False + + @property + def has_checks(self) -> bool: + """Whether the item opts into any check beyond ``tool_selection`` and ``tool_correctness``.""" + return bool( + self.expected_objects + or self.expect_widget is not None + or self.required_tools + or self.forbidden_tools + or self.answer_rubric + or self.expected_limit is not None + or self.no_raw_ids + or self.setup_turns + or not self.search_required + ) + + +def _string_list(key: str, value: Any) -> tuple[str, ...]: + # isinstance, not str(entry): an unquoted YAML `no` loads as False and would turn into the text "False". + if not isinstance(value, list) or not value: + raise ValueError(f"{key} must be a non-empty list of strings, got {value!r}") + if any(not isinstance(entry, str) or not entry.strip() for entry in value): + raise ValueError(f"{key} must hold non-empty strings only, got {value!r}") + return tuple(value) + + +def _flag(key: str, value: Any) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{key} must be true or false, got {value!r}") + return value + + +def _object_ref(ref: str) -> tuple[str, str]: + object_type, sep, object_id = ref.partition("/") + if not sep or object_type not in _OBJECT_TYPES or not object_id or object_id != object_id.strip(): + raise ValueError(f"expected_objects entry must be '/' with a known type, got {ref!r}") + return object_type, object_id + + +def _tool_args(tool_call: Any) -> dict[str, Any]: + if not isinstance(tool_call, dict): + raise ValueError(f"tool_call must be an object, got {tool_call!r}") + name = tool_call.get("function_name") + if name is not None and name != SEARCH_TOOL: + raise ValueError(f"tool_call.function_name must be {SEARCH_TOOL!r}, got {name!r}") + return check_arguments(tool_call.get("function_arguments", {})) + + +def check_arguments(args: Any) -> dict[str, Any]: + """The expected ``search_objects`` arguments, whichever way the item gives them. + + Raises: + ValueError: The arguments are not an object, or ``compatible_with_afm`` is not lists of objects. + """ + if not isinstance(args, dict): + raise ValueError(f"tool_call.function_arguments must be an object, got {args!r}") + afm = args.get("compatible_with_afm") + if afm is not None and ( + not isinstance(afm, dict) + or not set(afm) <= {"measures", "attributes"} + or any(value is not None and not isinstance(value, list) for value in afm.values()) + or any(not isinstance(item, dict) for value in afm.values() for item in value or []) + ): + raise ValueError(f"compatible_with_afm must hold lists of measure and attribute objects, got {afm!r}") + return dict(args) + + +def parse_expectation(expected_output: Any) -> SearchExpectation: + """Read an item's whole expected output. + + Raises: + ValueError: The expected output is not an object, names an unknown key, or holds a value the + run could not score. Raised before any request is sent. + """ + if not isinstance(expected_output, dict): + raise ValueError(f"expected_output must be an object, got {expected_output!r}") + unknown = sorted(set(expected_output) - _KNOWN_KEYS) + if unknown: + raise ValueError(f"expected_output has unknown keys {unknown}; known keys are {sorted(_KNOWN_KEYS)}") + raw = expected_output + tool_args = _tool_args(raw["tool_call"]) if "tool_call" in raw else {} + + expected_objects: tuple[str, ...] = () + if "expected_objects" in raw: + expected_objects = _string_list("expected_objects", raw["expected_objects"]) + for ref in expected_objects: + _object_ref(ref) + required = _string_list("required_tools", raw["required_tools"]) if "required_tools" in raw else () + forbidden = _string_list("forbidden_tools", raw["forbidden_tools"]) if "forbidden_tools" in raw else () + both = sorted(set(required) & set(forbidden)) + if both: + raise ValueError(f"tools {both} are both required and forbidden") + limit = raw.get("expected_limit") + if "expected_limit" in raw and (not isinstance(limit, int) or isinstance(limit, bool) or limit < 1): + raise ValueError(f"expected_limit must be a positive integer, got {limit!r}") + + spec = SearchExpectation( + tool_args=tool_args, + expected_objects=expected_objects, + expect_widget=_flag("expect_widget", raw["expect_widget"]) if "expect_widget" in raw else None, + required_tools=required, + forbidden_tools=forbidden, + search_required=_flag("search_required", raw["search_required"]) if "search_required" in raw else True, + answer_rubric=_string_list("answer_rubric", raw["answer_rubric"]) if "answer_rubric" in raw else (), + expected_limit=limit, + no_raw_ids=_flag("no_raw_ids", raw["no_raw_ids"]) if "no_raw_ids" in raw else False, + setup_turns=_string_list("setup_turns", raw["setup_turns"]) if "setup_turns" in raw else (), + report_only=_flag("report_only", raw["report_only"]) if "report_only" in raw else False, + ) + if "tool_call" not in raw and not spec.has_checks: + raise ValueError("expected_output asserts nothing: give tool_call or at least one check") + if spec.report_only and not spec.has_checks: + raise ValueError("report_only has no check to report") + return spec + + +def _ref_key(object_type: str, object_id: str) -> tuple[str, str]: + return ("attribute" if object_type in _ATTRIBUTE_TYPES else object_type, object_id) + + +def call_args(tc: ToolCallEvent) -> dict[str, Any]: + """The call's arguments, or {} when they are missing or not a JSON object.""" + args = tc.parsed_arguments() + return args if isinstance(args, dict) else {} + + +def search_calls(events: list[ToolCallEvent]) -> list[ToolCallEvent]: + """The turn's ``search_objects`` calls, in order.""" + return [tc for tc in events if tc.function_name == SEARCH_TOOL] + + +def _objects(container: Any) -> list[dict]: + """The object entries of a search result or widget part; none when its shape is not the expected one.""" + objects = container.get("objects") if isinstance(container, dict) else None + return [obj for obj in objects if isinstance(obj, dict)] if isinstance(objects, list) else [] + + +def returned_refs(result: ChatResult) -> list[str]: + """Every object the turn's searches returned, as ``type/id``, first sighting first. + + Read from the ``search_objects`` results and from the ``searchResults`` widget parts: a search + the agent runs for itself shows no widget, so its result is the only place its objects appear. + """ + refs: dict[tuple[str, str], str] = {} + + def _add(object_type: Any, object_id: Any) -> None: + if isinstance(object_type, str) and isinstance(object_id, str) and object_type and object_id: + refs.setdefault(_ref_key(object_type, object_id), f"{object_type}/{object_id}") + + for tc in search_calls(result.tool_call_events or []): + for obj in _objects(tc.parsed_result()): + _add(obj.get("final_type") or obj.get("type"), obj.get("id")) + for part in result.search_results: + for obj in _objects(part): + _add(obj.get("type"), obj.get("id")) + return list(refs.values()) + + +def _key_of(ref: str) -> tuple[str, str]: + object_type, _, object_id = ref.partition("/") + return _ref_key(object_type, object_id) + + +def missing_objects(expected: tuple[str, ...], returned: list[str]) -> list[str]: + """The expected refs no search returned. Every one must be found, not any one.""" + found = {_key_of(ref) for ref in returned} + return [ref for ref in expected if _key_of(ref) not in found] + + +def raw_ids(text: str) -> list[str]: + """The UUIDs and ``type/id`` spellings in ``text`` that sit outside a ``{type/id}`` chip.""" + outside = _CHIP.sub(" ", text) + typed = [m.group(0) for m in _TYPED_ID.finditer(outside) if _ID_MARK.search(m.group(1))] + return _UUID.findall(outside) + typed + + +def deterministic_checks(spec: SearchExpectation, result: ChatResult) -> tuple[dict[str, bool], dict[str, Any]]: + """The checks of one scored turn that need no judge, with the evidence of each failure.""" + checks: dict[str, bool] = {} + details: dict[str, Any] = {} + + def _check(name: str, ok: bool, why: Any) -> None: + checks[name] = ok + if not ok: + details[name] = why + + events = result.tool_call_events or [] + if spec.expected_objects: + returned = returned_refs(result) + missing = missing_objects(spec.expected_objects, returned) + _check("search_expected_objects", not missing, {"missing": missing, "returned": returned[:30]}) + if spec.expect_widget is not None: + parts = len(result.search_results) + _check("search_widget", (parts > 0) == spec.expect_widget, {"expected": spec.expect_widget, "parts": parts}) + if spec.required_tools or spec.forbidden_tools: + called = [tc.function_name for tc in events] + not_called = [name for name in spec.required_tools if name not in called] + forbidden = [name for name in spec.forbidden_tools if name in called] + _check( + "search_routing", + not not_called and not forbidden, + {"not_called": not_called, "forbidden_called": forbidden, "called": called}, + ) + if spec.expected_limit is not None: + limits = [call_args(tc).get("limit") for tc in search_calls(events)] + _check("search_limit", spec.expected_limit in limits, {"expected": spec.expected_limit, "limits": limits}) + if spec.no_raw_ids: + found = raw_ids(result.text_response or "") + _check("search_no_raw_ids", not found, {"raw_ids": found}) + return checks, details diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index e58a08dd3..9c969e018 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -1,21 +1,42 @@ # (C) 2026 GoodData Corporation. All rights reserved. -"""Agentic search-tool evaluation runner (single-turn).""" +"""Agentic search-tool evaluation runner (one scored turn, after any setup turns).""" from __future__ import annotations import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from typing import Any + +import httpx from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + JudgeErrorOf, + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) +from gooddata_eval.core.agentic._search_expectation import ( + SEARCH_TOOL, + SearchExpectation, + call_args, + check_arguments, + deterministic_checks, + parse_expectation, + search_calls, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -25,23 +46,61 @@ submit_trace_scoring, utc_now, ) -from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators._llm_judge import JudgeVerdict, LLMJudge, score_run from gooddata_eval.core.langfuse.item_scope import item_scope from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, + ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, ) +_TRANSPORT_ERRORS: tuple[type[BaseException], ...] = (httpx.HTTPError, TimeoutError, ConnectionError) + + +def _judge_provider_errors() -> tuple[type[BaseException], ...]: + """The errors of a judge request that fails at the provider (timeout, outage, rate limit). + + Such a request leaves the run unscored, not failed. ``openai`` is the optional llm-judge extra, so its + errors join only when it is installed. + """ + try: + from openai import OpenAIError # noqa: PLC0415 + except ImportError: + return _TRANSPORT_ERRORS + return (OpenAIError, *_TRANSPORT_ERRORS) + + _DEFAULT_K = 1 +# Every check score an item can publish beside tool_selection and tool_correctness. Each is published only for an +# item that opts into it, and search_answer not for a run the judge could not grade. +SEARCH_CONDITIONAL_SCORES = ( + "search_expected_objects", + "search_widget", + "search_routing", + "search_answer", + "search_limit", + "search_no_raw_ids", +) + +_RUBRIC_STEPS = [ + "EXPECTED OUTPUT lists the criteria, one per line, that the assistant's LAST reply must meet.", + "ACTUAL OUTPUT holds the conversation in order. Earlier turns are context only.", + "A {type/id} chip in a reply is shown to the user as that object's title.", + "Pass only when the last reply meets every criterion. Fail when it breaks any.", +] + def _tool_selection(tool_call_events: list[ToolCallEvent]) -> bool: """Return True if search_objects was called.""" - return any(tc.function_name == "search_objects" for tc in tool_call_events) + return any(tc.function_name == SEARCH_TOOL for tc in tool_call_events) # Arguments that do not determine search quality, so a fixture naming them must not @@ -77,30 +136,54 @@ def _list_matches(expected: list, actual: list, *, loose: bool) -> bool: return all(e in actual for e in expected) +def _afm_matches(expected: dict, actual: Any) -> bool: + """Every expected measure and attribute is in the call's AFM; a null field in the fixture matches anything. + + The agent states ``aggregation: null`` for a metric and may add objects to the AFM, so neither fails it. + """ + if not isinstance(actual, dict): + return False + if any(actual.get(key) is not None and not isinstance(actual.get(key), list) for key in ("measures", "attributes")): + return False + for key in ("measures", "attributes"): + listed = actual.get(key) + actual_items = [item for item in listed if isinstance(item, dict)] if isinstance(listed, list) else [] + for exp in expected.get(key) or []: + if not isinstance(exp, dict): + return False + if not any(all(v is None or act.get(k) == v for k, v in exp.items()) for act in actual_items): + return False + return True + + +def _call_matches(args: dict, expected_tool_call: dict) -> bool: + for key, exp_val in expected_tool_call.items(): + if key in _IGNORED_SEARCH_ARGS: + continue + act_val = args.get(key) + if key == "compatible_with_afm" and isinstance(exp_val, dict): + if not _afm_matches(exp_val, act_val): + return False + elif isinstance(exp_val, list) and isinstance(act_val, list): + if not _list_matches(exp_val, act_val, loose=key == "keywords"): + return False + elif isinstance(exp_val, str) and isinstance(act_val, str): + if not _loosely_contains(exp_val, act_val): + return False + elif exp_val != act_val: + return False + return True + + def _tool_correctness(tool_call_events: list[ToolCallEvent], expected_tool_call: dict) -> bool: - """Return True if the search_objects call arguments match expected. + """Return True if a search_objects call's arguments match expected. List fields (keywords, object_types) use subset matching: every expected value must appear in the actual call and the agent may include extras. ``keywords`` match by - substring; every other list is compared exactly. + substring; every other list is compared exactly. Any call of the turn may match: the + agent often looks the measure up first and asks for compatible attributes after. """ - for tc in tool_call_events: - if tc.function_name == "search_objects": - args = tc.parsed_arguments() or {} - for key, exp_val in expected_tool_call.items(): - if key in _IGNORED_SEARCH_ARGS: - continue - act_val = args.get(key) - if isinstance(exp_val, list) and isinstance(act_val, list): - if not _list_matches(exp_val, act_val, loose=key == "keywords"): - return False - elif isinstance(exp_val, str) and isinstance(act_val, str): - if not _loosely_contains(exp_val, act_val): - return False - elif exp_val != act_val: - return False - return True - return False + return any(_call_matches(call_args(tc), expected_tool_call) for tc in search_calls(tool_call_events)) @dataclass @@ -118,6 +201,21 @@ class SearchResult: # This run's own wall time -- mirrors the single-shot path's best_run_latency_s # (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 + # The item's opt-in checks of this run, and why each failed one failed. + checks: dict[str, bool] = field(default_factory=dict) + check_details: dict[str, Any] = field(default_factory=dict) + # Why the judge could not grade the answer; the run is then labelled ERROR unless another check failed it. + harness_error: str | None = None + # The checks that read False only because of harness_error. Their scores are not published. + harness_checks: tuple[str, ...] = () + # Set when a setup turn failed, so the question was never sent. + exit_reason: LoopExit | None = None + passed: bool = False + + @property + def judge_error(self) -> str | None: + """The name ``build_failed_runs`` reads a run's ungraded verdict under.""" + return self.harness_error @dataclass @@ -130,19 +228,148 @@ class AgenticSearchSummary: best: SearchResult +def _judge_rubric(judge: LLMJudge, rubric: tuple[str, ...], transcript: list[tuple[str, str]]) -> JudgeVerdict: + """Grade the last reply against the rubric, with the turns before it as context.""" + conversation = "\n\n".join(f"{role}: {text}" for role, text in transcript) + criteria = "\n".join(f"- {criterion}" for criterion in rubric) + question = next((text for role, text in reversed(transcript) if role == "User"), "") + return score_run(judge, input=question, expected_output=criteria, actual_output=conversation) + + +def _setup_failed(conversation_id: str, exit_reason: LoopExit, reason: str, latency_s: float) -> SearchResult: + """A run whose setup turn failed: the agent's FAIL, as a chat failure is for every kind. + + Its latency is the setup turns' own: the question was never sent. + """ + return SearchResult( + conversation_id=conversation_id, + tool_selected=False, + tool_correct=False, + tool_call_names=[], + check_details={"setup_turn": reason}, + exit_reason=exit_reason, + run_latency_s=latency_s, + ) + + +def _execute_run( + client: ChatClient, + conversation_id: str, + question: str, + spec: SearchExpectation, + judge: LLMJudge | None, +) -> SearchResult: + """Send the item's setup turns, then the question, and score the question's turn alone.""" + transcript: list[tuple[str, str]] = [] + setup_started = time.monotonic() + for number, turn in enumerate(spec.setup_turns, start=1): + try: + setup = client.send_message(conversation_id, turn) + except ChatError as exc: + reason = f"setup turn {number} failed: {exc}" + return _setup_failed(conversation_id, chat_failure_exit(exc), reason, time.monotonic() - setup_started) + setup_text = render_answer_text(setup) + if not setup_text and not setup.tool_call_events: + reason = f"setup turn {number} got no answer" + return _setup_failed(conversation_id, LoopExit.AGENT_SILENT, reason, time.monotonic() - setup_started) + transcript += [("User", turn), ("Assistant", setup_text)] + + run_started = time.monotonic() + chat_result = client.send_message(conversation_id, question) + run_latency = time.monotonic() - run_started + tcs = chat_result.tool_call_events or [] + selected = _tool_selection(tcs) + run = SearchResult( + conversation_id=conversation_id, + tool_selected=selected, + tool_correct=selected and _tool_correctness(tcs, spec.tool_args), + tool_call_names=[tc.function_name for tc in tcs], + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), + run_latency_s=run_latency, + ) + if spec.has_checks: + run.checks, run.check_details = deterministic_checks(spec, chat_result) + if spec.answer_rubric: + _score_answer(run, spec, judge, transcript, question, chat_result) + run.passed = (selected or not spec.search_required) and (spec.report_only or all(run.checks.values())) + return run + + +def _score_answer( + run: SearchResult, + spec: SearchExpectation, + judge: LLMJudge | None, + transcript: list[tuple[str, str]], + question: str, + chat_result: ChatResult, +) -> None: + """Grade the reply's text against the rubric. A turn without text fails without asking the judge.""" + answer = (chat_result.text_response or "").strip() + if not answer: + run.checks["search_answer"] = False + run.check_details["search_answer"] = "empty answer: the turn has no text" + return + if judge is None: + raise RuntimeError("answer_rubric needs the LLM judge, which was not built") + try: + verdict = _judge_rubric(judge, spec.answer_rubric, [*transcript, ("User", question), ("Assistant", answer)]) + except _judge_provider_errors() as exc: + verdict = JudgeVerdict(passed=False, reasoning="", error=f"{type(exc).__name__}: {exc}") + run.checks["search_answer"] = verdict.passed if verdict.error is None else False + if verdict.error is not None: + reason = f"judge gave no verdict: {verdict.error}" + run.check_details["search_answer"] = f"harness: {reason}" + run.harness_error = reason + run.harness_checks = ("search_answer",) + elif not verdict.passed: + run.check_details["search_answer"] = verdict.reasoning + + +def _open_harness_error(spec: SearchExpectation) -> JudgeErrorOf: + """How labels read a run's harness error: only when nothing but the harness-scored checks failed it. + + A run that also failed a check the harness did score is the agent's FAIL, whatever the judge would have said. + """ + + def _read(run: Any) -> str | None: + if run.harness_error is None: + return None + gate_ok = run.tool_selected or not spec.search_required + failing = {name for name, ok in run.checks.items() if not ok} + return run.harness_error if gate_ok and failing <= set(run.harness_checks) else None + + return _read + + def run_agentic_search_tool( host: str, token: str, workspace_id: str, question: str, - expected_tool_call: dict, + expected_tool_call: dict | None = None, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, user_context: dict | None = None, + expected_output: dict | None = None, ) -> AgenticSearchSummary: - """Run the search-tool agentic evaluation K times (single-turn each).""" + """Run the search-tool agentic evaluation K times. + + ``expected_tool_call`` is the bare ``search_objects`` arguments. ``expected_output`` is the item's whole expected output (``tool_call`` plus the checks of + ``_search_expectation``); when it has no ``tool_call``, ``expected_tool_call`` supplies the arguments. + + Raises: + ValueError: ``expected_output`` or ``expected_tool_call`` is malformed. Raised before any request. + ImportError, OSError: The item has an answer rubric and the LLM judge cannot be built (the llm-judge + extra or OPENAI_API_KEY is missing). Raised before any request. + """ + spec = _expectation(expected_tool_call, expected_output) + # Built before the first request: a missing judge is a setup error to report at once. + judge = LLMJudge(_RUBRIC_STEPS) if spec.answer_rubric else None run_results: list[SearchResult] = [] client = ChatClient( @@ -156,24 +383,7 @@ def run_agentic_search_tool( try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - run_started = time.monotonic() - chat_result = client.send_message(conv_id_0, question) - tcs = chat_result.tool_call_events or [] - selected = _tool_selection(tcs) - correct = selected and _tool_correctness(tcs, expected_tool_call) - run_results.append( - SearchResult( - conversation_id=conv_id_0, - tool_selected=selected, - tool_correct=correct, - tool_call_names=[tc.function_name for tc in tcs], - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - tool_call_events=list(chat_result.tool_call_events or []), - reasoning_step_events=list(chat_result.reasoning_step_events or []), - run_latency_s=time.monotonic() - run_started, - ) - ) + run_results.append(_execute_run(client, conv_id_0, question, spec, judge)) finally: if initial_conversation_id is None: client.delete_conversation(conv_id_0) @@ -181,34 +391,15 @@ def run_agentic_search_tool( for _ in range(1, k): conv_id = client.create_conversation() try: - run_started = time.monotonic() - chat_result = client.send_message(conv_id, question) - tcs = chat_result.tool_call_events or [] - selected = _tool_selection(tcs) - correct = selected and _tool_correctness(tcs, expected_tool_call) - run_results.append( - SearchResult( - conversation_id=conv_id, - tool_selected=selected, - tool_correct=correct, - tool_call_names=[tc.function_name for tc in tcs], - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - tool_call_events=list(chat_result.tool_call_events or []), - reasoning_step_events=list(chat_result.reasoning_step_events or []), - run_latency_s=time.monotonic() - run_started, - ) - ) + run_results.append(_execute_run(client, conv_id, question, spec, judge)) finally: client.delete_conversation(conv_id) finally: client.close() - # Pass requires only tool_selected — tool_correct is a Langfuse quality metric. - # This matches the original Tavern behavior where only missing the tool call failed the test. - pass_at_k = any(r.tool_selected for r in run_results) - pass_power_k = all(r.tool_selected for r in run_results) - best = max(run_results, key=lambda r: (r.tool_correct, r.tool_selected)) + pass_at_k = any(r.passed for r in run_results) + pass_power_k = all(r.passed for r in run_results) + best = max(run_results, key=lambda r: (r.passed, sum(r.checks.values()), r.tool_correct, r.tool_selected)) return AgenticSearchSummary( run_results=run_results, pass_at_k=pass_at_k, @@ -217,23 +408,52 @@ def run_agentic_search_tool( ) +def _expectation(expected_tool_call: dict | None, expected_output: dict | None) -> SearchExpectation: + """The item's expectation from either calling convention. + + Raises: + ValueError: ``expected_output`` or ``expected_tool_call`` is malformed. + """ + if expected_output is None: + return SearchExpectation(tool_args=check_arguments(expected_tool_call or {})) + spec = parse_expectation(expected_output) + if "tool_call" in expected_output or not expected_tool_call: + return spec + return replace(spec, tool_args=check_arguments(expected_tool_call)) + + class SearchToolAssertionError(AgenticAssertionError): """Raised when a search-tool evaluation fails.""" -def _run_detail(run: SearchResult) -> dict: - """The diagnostic fields for ONE run, shared by the best run and every failing one. +def _gating_checks(run: SearchResult, spec: SearchExpectation) -> dict[str, bool]: + """The run's checks that decide it: none for a report-only item, and never one the judge could not grade.""" + if spec.report_only: + return {} + return {name: ok for name, ok in run.checks.items() if name not in run.harness_checks} + - Extracted so a failing run is described by exactly the same keys as the winning run -- - for this kind that means the tools it actually called, which is the whole question when - the run failed because it selected the wrong one. +def _verdict_fields(run: SearchResult, spec: SearchExpectation) -> dict[str, Any]: + """The run's gating verdicts as top-level booleans of its detail, which the report's quality score counts. + + What does not gate sits under ``diagnostics``: the search verdicts of an item that needs no search, and + the checks of a report-only item. """ - return { - "tool_selected": run.tool_selected, - "tool_correct": run.tool_correct, - "tool_call_names": run.tool_call_names, - "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), - } + searched = {"tool_selected": run.tool_selected, "tool_correct": run.tool_correct} + fields: dict[str, Any] = dict(searched) if spec.search_required else {} + fields.update(_gating_checks(run, spec)) + diagnostics: dict[str, bool] = {} if spec.search_required else dict(searched) + if spec.report_only: + diagnostics.update(run.checks) + if diagnostics: + fields["diagnostics"] = diagnostics + if run.check_details: + fields["check_details"] = run.check_details + if run.harness_error: + fields["harness_error"] = run.harness_error + if run.exit_reason is not None: + fields["exit_reason"] = run.exit_reason.value + return fields def evaluate_agentic_search_tool( @@ -241,7 +461,7 @@ def evaluate_agentic_search_tool( token: str, workspace_id: str, question: str, - expected_tool_call: dict, + expected_tool_call: dict | None = None, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -254,6 +474,7 @@ def evaluate_agentic_search_tool( reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, user_context: dict | None = None, + expected_output: dict | None = None, gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure. @@ -261,7 +482,12 @@ def evaluate_agentic_search_tool( Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an AgenticEvalOutcome on success; on failure the same three values are attached to the raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. + ``expected_tool_call`` and ``expected_output`` are read as in ``run_agentic_search_tool``. + + Raises: + ValueError: ``expected_output`` or ``expected_tool_call`` is malformed. Raised before any request. """ + spec = _expectation(expected_tool_call, expected_output) identity = RunIdentity( host, token, @@ -285,8 +511,25 @@ def evaluate_agentic_search_tool( reasoning_effort=reasoning_effort, agent_id=agent_id, user_context=user_context, + expected_output=expected_output, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.passed, judge_error=_open_harness_error(spec)) + + def _strict(run: SearchResult) -> dict[str, bool]: + strict = {"tool_selection": run.tool_selected} if spec.search_required else {} + strict.update(_gating_checks(run, spec)) + return strict or {"passed": run.passed} + + def _run_detail(run: SearchResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one.""" + return { + **_verdict_fields(run, spec), + "tool_call_names": run.tool_call_names, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -299,12 +542,18 @@ def _write_scores(ctx: RunTraceContext) -> None: with ctx.observe( pt, run_idx, conversation_id=run.conversation_id, output={"tool_selection": run.tool_selected} ) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) ctx.score(tid, name="tool_selection", value=float(run.tool_selected), data_type="BOOLEAN") - ctx.score(tid, name="tool_correctness", value=float(run.tool_correct), data_type="BOOLEAN") + if spec.tool_args: + ctx.score(tid, name="tool_correctness", value=float(run.tool_correct), data_type="BOOLEAN") + for name, ok in run.checks.items(): + if name not in run.harness_checks: + ctx.score(tid, name=name, value=float(ok), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, - strict_checks={"tool_selection": run.tool_selected}, + strict_checks=_strict(run), latency_sec=pt.latency if pt else None, cost_usd=pt.total_cost if pt else None, ) @@ -324,32 +573,50 @@ def _write_scores(ctx: RunTraceContext) -> None: item_input=question, ) - runs_passed = sum(1 for r in summary.run_results if r.tool_selected) + runs_passed = sum(1 for r in summary.run_results if r.passed) runs_effective = len(summary.run_results) + def _failed_checks(run: SearchResult) -> list[dict]: + if run.exit_reason is not None: + return failed_check_records([FailedCheck("setup_turn", actual=run.check_details.get("setup_turn"))]) + checks: list[FailedCheck] = [] + if spec.search_required and not run.tool_selected: + checks.append(FailedCheck("tool_selection", expected=True, actual=run.tool_call_names)) + checks += [ + FailedCheck(name, expected=True, actual=run.check_details.get(name, False)) + for name, ok in {**_gating_checks(run, spec), **{n: run.checks[n] for n in run.harness_checks}}.items() + if not ok + ] + return failed_check_records(checks) + best = summary.best detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, - tool_selected=best.tool_selected, - tool_correct=best.tool_correct, + **_verdict_fields(best, spec), tool_call_names=best.tool_call_names, ) # Same predicate pass@K and runs_passed are taken over, so an item's failed_runs and its # counts cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.tool_selected, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + ungraded = sum(1 for r in summary.run_results if _open_harness_error(spec)(r) is not None) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, ungraded) + failing = [record["name"] for record in _failed_checks(best)] + checks_str = f" Failed checks: {', '.join(failing)}; details: {best.check_details}." if failing else "" raise_agentic_failure( SearchToolAssertionError, f"Search tool assertion failed. {gate_note} " f"tool_selected={best.tool_selected}, tool_correct={best.tool_correct}. " - f"Tool calls made: {best.tool_call_names}", + f"Tool calls made: {best.tool_call_names}.{checks_str}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -361,6 +628,7 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s=[r.run_latency_s for r in summary.run_results], ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/tests/test_agentic_search_tool_checks.py b/packages/gooddata-eval/tests/test_agentic_search_tool_checks.py new file mode 100644 index 000000000..5d2fd8886 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_search_tool_checks.py @@ -0,0 +1,676 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +"""The opt-in search-tool checks. Response shapes are trimmed from runs on the ecommerce_demo fixture.""" + +import json +import sys +from contextlib import contextmanager +from dataclasses import fields +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.core.agentic._search_expectation import ( + SEARCH_OPTION_KEYS, + SearchExpectation, + deterministic_checks, + parse_expectation, + raw_ids, + returned_refs, +) +from gooddata_eval.core.agentic.search_tool import ( + SEARCH_CONDITIONAL_SCORES, + SearchToolAssertionError, + _judge_provider_errors, + _tool_correctness, + evaluate_agentic_search_tool, + run_agentic_search_tool, +) +from gooddata_eval.core.chat.sse_client import ChatError, TurnTimeoutError +from gooddata_eval.core.evaluators._llm_judge import JudgeVerdict +from gooddata_eval.core.models import ChatResult, DatasetItem + +_MODULE = "gooddata_eval.core.agentic.search_tool" +_HOST = "http://host" + + +def _search_call(args: dict, objects: list[dict] | None = None) -> dict: + result = {"objects": objects or [], "returned_count": len(objects or []), "search_results_ref": "search_1"} + return {"functionName": "search_objects", "functionArguments": json.dumps(args), "result": json.dumps(result)} + + +def _obj(object_type: str, object_id: str, final_type: str | None = None) -> dict: + return {"id": object_id, "type": object_type, "final_type": final_type or object_type, "title": object_id} + + +def _widget(*objects: tuple[str, str]) -> dict: + return { + "type": "searchResults", + "objects": [{"id": object_id, "type": object_type, "title": object_id} for object_type, object_id in objects], + "requestedObjectType": objects[0][0] if objects else "metric", + } + + +def _turn( + text: str = "Found 2 results.", + calls: list[dict] | None = None, + widgets: list[dict] | None = None, +) -> ChatResult: + return ChatResult.model_validate( + {"textResponse": text, "toolCallEvents": calls or [], "searchResults": widgets or [], "responseId": "resp"} + ) + + +_NET_SALES_CALL = _search_call( + {"keywords": ["Net Sales"], "object_types": ["metric", "fact"], "limit": 20, "compatible_with_afm": None}, + [_obj("metric", "net_sales"), _obj("metric", "total_sales")], +) +_COMPATIBLE_CALL = _search_call( + { + "keywords": None, + "object_types": ["attribute", "date_attribute"], + "limit": 100, + "compatible_with_afm": { + "measures": [{"id": "net_sales", "type": "metric", "aggregation": None}], + "attributes": [], + }, + }, + [ + _obj("attribute", "customer_created_date.dayOfMonth", "date_attribute"), + _obj("attribute", "product_category"), + ], +) + + +def _client(*turns: Any) -> MagicMock: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = list(turns) + return client + + +def _run(expected_output: dict, *turns: Any, judge: Any = None) -> Any: + client = _client(*turns) + with patch(f"{_MODULE}.ChatClient", return_value=client), patch(f"{_MODULE}.LLMJudge", return_value=judge): + summary = run_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected_output + ) + # The scored turn is the question, sent last in the run's own conversation. + assert client.send_message.call_args.args == ("conv-1", "Find Net Sales") + return summary.best, client + + +def _evaluate(expected_output: dict, *turns: Any) -> Any: + client = _client(*turns) + with patch(f"{_MODULE}.ChatClient", return_value=client): + return evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected_output + ) + + +# --- parsing ------------------------------------------------------------------------------------- + + +def test_an_item_with_only_tool_call_has_no_checks(): + spec = parse_expectation({"tool_call": {"function_name": "search_objects", "function_arguments": {"limit": 10}}}) + assert spec.tool_args == {"limit": 10} + assert spec.has_checks is False + + +@pytest.mark.parametrize( + ("expected_output", "message"), + [ + ("search", "must be an object"), + ({"expect_widgets": True}, "unknown keys"), + ({"expected_objects": ["metrics/net_sales"]}, "known type"), + ({"expected_objects": ["metric/"]}, "known type"), + ({"expected_objects": "metric/net_sales"}, "non-empty list"), + ({"expected_objects": []}, "non-empty list"), + ({"answer_rubric": ["says more exist", " "]}, "non-empty strings"), + ({"setup_turns": [False]}, "non-empty strings"), + ({"expect_widget": "yes"}, "true or false"), + ({"no_raw_ids": 1}, "true or false"), + ({"expected_limit": True}, "positive integer"), + ({"expected_limit": 0}, "positive integer"), + ({"required_tools": ["search_objects"], "forbidden_tools": ["search_objects"]}, "both required and forbidden"), + ({"tool_call": {"function_name": "search_knowledge"}}, "function_name"), + ({"tool_call": "search_objects"}, "tool_call must be an object"), + ({"tool_call": {"function_arguments": ["Net Sales"]}}, "function_arguments must be an object"), + ({"tool_call": {"function_arguments": {"compatible_with_afm": {"measures": 0}}}}, "compatible_with"), + ({"tool_call": {"function_arguments": {"compatible_with_afm": {"metrics": []}}}}, "compatible_with_afm"), + ( + {"tool_call": {"function_arguments": {"compatible_with_afm": {"measures": ["net_sales"]}}}}, + "compatible_with", + ), + ({"tool_call": {"function_arguments": {"compatible_with_afm": {"measures": "net_sales"}}}}, "compatible_with"), + ({}, "asserts nothing"), + ({"tool_call": {"function_arguments": {"keywords": ["x"]}}, "report_only": True}, "no check to report"), + ], +) +def test_a_malformed_item_is_rejected(expected_output: Any, message: str): + with pytest.raises(ValueError, match=message): + parse_expectation(expected_output) + + +@pytest.mark.parametrize("afm", [{"measures": 0}, {"measures": 1}, {"attributes": True}, {"measures": ["net_sales"]}]) +@pytest.mark.parametrize("with_output", [False, True]) +def test_malformed_bare_arguments_fail_before_any_request(afm: dict, with_output: bool): + client = MagicMock() + with patch(f"{_MODULE}.ChatClient", return_value=client), pytest.raises(ValueError, match="compatible_with_afm"): + run_agentic_search_tool( + host=_HOST, + token="tok", + workspace_id="ws", + question="q", + expected_tool_call={"compatible_with_afm": afm}, + expected_output={"expect_widget": True} if with_output else None, + ) + client.send_message.assert_not_called() + + +def test_a_tool_call_without_arguments_is_an_item(): + spec = parse_expectation({"tool_call": {"function_arguments": {}}}) + assert spec.tool_args == {} + assert spec.has_checks is False + + +def test_a_malformed_item_fails_before_any_request(): + client = MagicMock() + with patch(f"{_MODULE}.ChatClient", return_value=client), pytest.raises(ValueError, match="unknown keys"): + evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="q", expected_output={"expected_object": ["x"]} + ) + client.send_message.assert_not_called() + + +def test_every_option_key_is_an_expectation_field(): + assert {f.name for f in fields(SearchExpectation)} >= SEARCH_OPTION_KEYS + + +# --- tool_correctness ---------------------------------------------------------------------------- + + +def test_tool_correctness_accepts_a_later_call(): + events = _turn(calls=[_NET_SALES_CALL, _COMPATIBLE_CALL]).tool_call_events + expected = {"compatible_with_afm": {"measures": [{"id": "net_sales"}]}} + assert _tool_correctness(events, expected) is True + + +def test_tool_correctness_compares_the_afm_loosely(): + events = _turn(calls=[_COMPATIBLE_CALL]).tool_call_events + assert _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales", "aggregation": None}]}}) + assert _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales", "type": "metric"}]}}) + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "sell_through"}]}}) + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales", "type": "fact"}]}}) + + +def test_tool_correctness_fails_bare_afm_ids_instead_of_crashing(): + events = _turn(calls=[_COMPATIBLE_CALL]).tool_call_events + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": ["net_sales"]}}) + + +def test_tool_correctness_fails_an_afm_collection_that_is_not_a_list(): + odd = _search_call({"compatible_with_afm": {"measures": 1, "attributes": None}}) + assert not _tool_correctness( + _turn(calls=[odd]).tool_call_events, {"compatible_with_afm": {"measures": [{"id": "x"}]}} + ) + attrs = _search_call({"compatible_with_afm": {"measures": 1, "attributes": [{"id": "product_category"}]}}) + expected = {"compatible_with_afm": {"attributes": [{"id": "product_category"}]}} + assert not _tool_correctness(_turn(calls=[attrs]).tool_call_events, expected) + + +def test_tool_correctness_reads_null_arguments_as_none(): + null_args = {"functionName": "search_objects", "functionArguments": "null", "result": "null"} + assert _tool_correctness(_turn(calls=[null_args]).tool_call_events, {}) is True + assert not _tool_correctness(_turn(calls=[null_args]).tool_call_events, {"keywords": ["Net Sales"]}) + + +def test_tool_correctness_fails_an_afm_the_agent_never_sent(): + events = _turn(calls=[_NET_SALES_CALL]).tool_call_events + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales"}]}}) + + +# --- deterministic checks ------------------------------------------------------------------------ + + +def test_expected_objects_are_read_from_an_internal_search_without_widget(): + result = _turn(calls=[_NET_SALES_CALL, _COMPATIBLE_CALL]) + checks, _ = deterministic_checks(parse_expectation({"expected_objects": ["attribute/product_category"]}), result) + assert checks == {"search_expected_objects": True} + + +def test_expected_objects_are_read_from_the_widget(): + result = _turn(widgets=[_widget(("dashboard", "370ec88c-6c35-4d2c-9e2d-3ef7cbd8e2f1"))]) + spec = parse_expectation({"expected_objects": ["dashboard/370ec88c-6c35-4d2c-9e2d-3ef7cbd8e2f1"]}) + assert deterministic_checks(spec, result)[0] == {"search_expected_objects": True} + + +def test_every_expected_object_must_be_returned(): + result = _turn(calls=[_NET_SALES_CALL]) + spec = parse_expectation({"expected_objects": ["metric/net_sales", "metric/total_customers"]}) + checks, details = deterministic_checks(spec, result) + assert checks == {"search_expected_objects": False} + assert details["search_expected_objects"]["missing"] == ["metric/total_customers"] + + +def test_an_attribute_matches_by_any_of_its_spellings(): + result = _turn(calls=[_COMPATIBLE_CALL]) + for ref in ("attribute/customer_created_date.dayOfMonth", "date_attribute/customer_created_date.dayOfMonth"): + assert deterministic_checks(parse_expectation({"expected_objects": [ref]}), result)[0][ + "search_expected_objects" + ] + assert deterministic_checks(parse_expectation({"expected_objects": ["label/product_category"]}), result)[0][ + "search_expected_objects" + ] + + +def test_an_object_type_is_not_ignored(): + result = _turn(calls=[_NET_SALES_CALL]) + assert not deterministic_checks(parse_expectation({"expected_objects": ["fact/net_sales"]}), result)[0][ + "search_expected_objects" + ] + + +def test_returned_objects_of_other_synced_types_are_read(): + date_dataset = _search_call({"keywords": ["date"]}, [_obj("dataset", "dt_order", "date"), _obj("parameter", "p_1")]) + result = _turn(calls=[date_dataset]) + spec = parse_expectation({"expected_objects": ["date/dt_order", "metric/net_sales"]}) + checks, details = deterministic_checks(spec, result) + assert details["search_expected_objects"]["missing"] == ["metric/net_sales"] + assert returned_refs(result) == ["date/dt_order", "parameter/p_1"] + + +def test_a_result_that_is_not_an_object_returns_nothing(): + odd = {"functionName": "search_objects", "functionArguments": "null", "result": '"no objects"'} + result = _turn(calls=[odd]) + assert returned_refs(result) == [] + checks, _ = deterministic_checks(parse_expectation({"expected_objects": ["metric/x"], "expected_limit": 5}), result) + assert checks == {"search_expected_objects": False, "search_limit": False} + + +def test_objects_that_are_not_a_list_return_nothing(): + odd = {"functionName": "search_objects", "functionArguments": "{}", "result": '{"objects": 1}'} + result = _turn(calls=[odd], widgets=[{"type": "searchResults", "objects": 1}]) + assert returned_refs(result) == [] + + +def test_returned_refs_keep_the_first_spelling_and_order(): + result = _turn(calls=[_COMPATIBLE_CALL], widgets=[_widget(("attribute", "customer_created_date.dayOfMonth"))]) + assert returned_refs(result) == ["date_attribute/customer_created_date.dayOfMonth", "attribute/product_category"] + + +@pytest.mark.parametrize( + ("expect", "widgets", "ok"), [(True, 1, True), (True, 0, False), (False, 0, True), (False, 1, False)] +) +def test_widget(expect: bool, widgets: int, ok: bool): + result = _turn(widgets=[_widget(("metric", "net_sales"))] * widgets) + assert deterministic_checks(parse_expectation({"expect_widget": expect}), result)[0] == {"search_widget": ok} + + +def test_routing_needs_every_required_tool_and_no_forbidden_one(): + describe = {"functionName": "get_object_with_children", "functionArguments": "{}"} + result = _turn(calls=[_NET_SALES_CALL, describe]) + ok = parse_expectation({"required_tools": ["search_objects", "get_object_with_children"]}) + assert deterministic_checks(ok, result)[0] == {"search_routing": True} + knowledge = parse_expectation({"forbidden_tools": ["search_knowledge", "get_object_with_children"]}) + checks, details = deterministic_checks(knowledge, result) + assert checks == {"search_routing": False} + assert details["search_routing"]["forbidden_called"] == ["get_object_with_children"] + + +def test_limit_passes_when_any_call_asked_for_it(): + retry = _search_call({"keywords": None, "object_types": ["metric"], "limit": 100}) + first = _search_call({"keywords": None, "object_types": ["metric"], "limit": 20}) + result = _turn(calls=[first, retry]) + assert deterministic_checks(parse_expectation({"expected_limit": 20}), result)[0] == {"search_limit": True} + assert deterministic_checks(parse_expectation({"expected_limit": 50}), result)[0] == {"search_limit": False} + + +@pytest.mark.parametrize( + ("text", "found"), + [ + ("Found 10 eligible metrics. `{metric/orders_all_time}` ignores dashboard filters.", []), + ("It is used in {visualization/03b89e08-ee06-4e1f-88d8-bbd805c691cc}.", []), + ("I searched metric/fact/attribute together.", []), + ("Use a date/time attribute, label/value pairs, the dataset/table or a metric/KPI.", []), + ("Search metric/revenue.", []), + ("The dataset is date/dt_order.", ["date/dt_order"]), + ("See 03b89e08-ee06-4e1f-88d8-bbd805c691cc.", ["03b89e08-ee06-4e1f-88d8-bbd805c691cc"]), + ("The id is metric/net_sales.", ["metric/net_sales"]), + ("Use attribute/customer_created_date.dayOfMonth.", ["attribute/customer_created_date.dayOfMonth"]), + ], +) +def test_raw_ids_allow_chips_only(text: str, found: list[str]): + assert raw_ids(text) == found + + +# --- runs ---------------------------------------------------------------------------------------- + + +def test_setup_turns_go_first_and_only_the_last_turn_is_scored(): + setup = _turn(text="Found 10 results for revenue.", calls=[_search_call({"keywords": ["revenue"]})]) + scored = _turn( + text="Found 5 dashboards.", + calls=[_search_call({"keywords": ["revenue"], "object_types": ["dashboard"]})], + widgets=[_widget(("dashboard", "092929af"))], + ) + expected = { + "tool_call": {"function_arguments": {"object_types": ["dashboard"]}}, + "setup_turns": ["Search for revenue"], + "expect_widget": True, + } + run, client = _run(expected, setup, scored) + assert [c.args[1] for c in client.send_message.call_args_list] == ["Search for revenue", "Find Net Sales"] + assert {c.args[0] for c in client.send_message.call_args_list} == {"conv-1"} + assert run.passed and run.tool_correct + assert len(run.tool_call_events) == 1 + + +def test_a_failed_setup_turn_records_its_own_latency(): + clock = MagicMock() + clock.monotonic.side_effect = [10.0, 12.5] + with patch(f"{_MODULE}.time", clock), pytest.raises(SearchToolAssertionError) as exc_info: + _evaluate({"setup_turns": ["Search for revenue"], "expect_widget": True}, TurnTimeoutError("too slow")) + assert exc_info.value.run_latencies_s == [2.5] + + +@pytest.mark.parametrize( + ("setup", "exit_reason"), + [ + (ChatError("stream broke"), "chat_error"), + (TurnTimeoutError("too slow"), "agent_timeout"), + (_turn(text=""), "agent_silent"), + ], +) +def test_a_failed_setup_turn_is_the_agents_fail(setup: Any, exit_reason: str): + with pytest.raises(SearchToolAssertionError) as exc_info: + _evaluate({"setup_turns": ["Search for revenue"], "expect_widget": True}, setup) + error = exc_info.value + run = error.failed_runs[0] + assert run["outcome"]["status"] == "fail" + assert run["exit_reason"] == exit_reason + assert [check["name"] for check in run["failed_checks"]] == ["setup_turn"] + assert not hasattr(error, "harness_error") + + +def test_search_required_false_passes_without_a_search(): + chart = {"functionName": "create_visualization", "functionArguments": "{}"} + run, _ = _run({"search_required": False, "expect_widget": False}, _turn(text="Here is the chart.", calls=[chart])) + assert run.passed + assert run.tool_selected is False + + +def test_a_failing_check_fails_the_run(): + run, _ = _run({"tool_call": {"function_arguments": {}}, "expect_widget": True}, _turn(calls=[_NET_SALES_CALL])) + assert run.tool_selected + assert not run.passed + + +def test_report_only_checks_do_not_gate(): + expected = {"expect_widget": True, "report_only": True} + outcome = _evaluate(expected, _turn(calls=[_NET_SALES_CALL])) + assert "search_widget" not in outcome.detail + assert outcome.detail["diagnostics"] == {"search_widget": False} + assert outcome.runs_passed == 1 + + +def test_a_run_that_rightly_skips_search_keeps_the_search_verdicts_out_of_quality(): + chart = {"functionName": "create_visualization", "functionArguments": "{}"} + outcome = _evaluate({"search_required": False, "expect_widget": False}, _turn(calls=[chart])) + booleans = {k: v for k, v in outcome.detail.items() if isinstance(v, bool)} + assert booleans == {"search_widget": True} + assert outcome.detail["diagnostics"] == {"tool_selected": False, "tool_correct": False} + + +def test_an_empty_answer_fails_the_rubric_without_asking_the_judge(): + judge = MagicMock() + with patch(f"{_MODULE}.score_run") as score: + run, _ = _run({"answer_rubric": ["Says more exist"]}, _turn(text="", calls=[_NET_SALES_CALL]), judge=judge) + score.assert_not_called() + assert run.checks == {"search_answer": False} + assert run.check_details["search_answer"].startswith("empty answer") + assert run.harness_error is None + + +def test_the_judge_grades_the_last_reply_with_the_setup_as_context(): + expected = {"answer_rubric": ["Names only 1. Overview"], "setup_turns": ["Describe Revenue Trend"]} + with patch(f"{_MODULE}.score_run", return_value=JudgeVerdict(passed=True, reasoning="ok")) as score: + run, _ = _run(expected, _turn(text="It shows revenue."), _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert run.checks == {"search_answer": True} + kwargs = score.call_args.kwargs + assert kwargs["expected_output"] == "- Names only 1. Overview" + assert kwargs["actual_output"].endswith("User: Find Net Sales\n\nAssistant: Found 2 results.") + assert kwargs["actual_output"].startswith("User: Describe Revenue Trend") + + +def test_a_failed_rubric_keeps_the_judges_reasoning(): + verdict = JudgeVerdict(passed=False, reasoning="does not say more exist") + with patch(f"{_MODULE}.score_run", return_value=verdict): + run, _ = _run({"answer_rubric": ["Says more exist"]}, _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert run.checks == {"search_answer": False} + assert run.check_details["search_answer"] == "does not say more exist" + assert run.harness_error is None + + +def test_a_judge_provider_error_leaves_the_answer_ungraded(): + with patch(f"{_MODULE}.score_run", side_effect=TimeoutError("judge timed out")): + run, _ = _run({"answer_rubric": ["Says more exist"]}, _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert run.harness_error == "judge gave no verdict: TimeoutError: judge timed out" + assert run.harness_checks == ("search_answer",) + + +def test_judge_provider_errors_need_no_llm_judge_extra(): + with patch.dict(sys.modules, {"openai": None}): + assert _judge_provider_errors() == (httpx.HTTPError, TimeoutError, ConnectionError) + assert len(_judge_provider_errors()) == 4 + + +def test_a_rubric_without_a_judge_is_a_programming_error(): + with pytest.raises(RuntimeError, match="LLM judge"): + _run({"answer_rubric": ["Says more exist"]}, _turn(calls=[_NET_SALES_CALL]), judge=None) + + +def test_a_judge_without_verdict_labels_the_run_error(): + expected = {"tool_call": {"function_arguments": {"keywords": ["Net Sales"]}}, "answer_rubric": ["Lists net sales"]} + verdict = JudgeVerdict(passed=False, reasoning="", error="unreadable") + client = _client(_turn(calls=[_NET_SALES_CALL])) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.LLMJudge"), + patch(f"{_MODULE}.score_run", return_value=verdict), + pytest.raises(SearchToolAssertionError) as exc_info, + ): + evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected + ) + run = exc_info.value.failed_runs[0] + assert run["outcome"]["status"] == "error" + assert "1 ungraded" in str(exc_info.value) + assert run["error"] == "judge gave no verdict: unreadable" + assert "search_answer" not in exc_info.value.detail + assert exc_info.value.detail["check_details"]["search_answer"].startswith("harness:") + + +def test_a_judge_error_is_not_blamed_on_the_harness_when_another_check_failed(): + expected = {"answer_rubric": ["Lists net sales"], "expect_widget": True} + verdict = JudgeVerdict(passed=False, reasoning="", error="unreadable") + client = _client(_turn(calls=[_NET_SALES_CALL])) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.LLMJudge"), + patch(f"{_MODULE}.score_run", return_value=verdict), + pytest.raises(SearchToolAssertionError) as exc_info, + ): + evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected + ) + assert exc_info.value.failed_runs[0]["outcome"]["status"] == "fail" + + +def test_expected_output_without_tool_call_takes_the_bare_arguments(): + client = _client(_turn(calls=[_NET_SALES_CALL])) + with patch(f"{_MODULE}.ChatClient", return_value=client): + summary = run_agentic_search_tool( + host=_HOST, + token="tok", + workspace_id="ws", + question="Find Net Sales", + expected_tool_call={"object_types": ["dashboard"]}, + expected_output={"expect_widget": False}, + ) + assert summary.best.tool_correct is False + assert summary.best.passed + + +def test_an_item_with_only_tool_call_keeps_the_legacy_detail(): + outcome = _evaluate( + {"tool_call": {"function_name": "search_objects", "function_arguments": {"keywords": ["Net Sales"]}}}, + _turn(calls=[_NET_SALES_CALL]), + ) + assert set(outcome.detail) == { + "tool_selected", + "tool_correct", + "tool_call_names", + "latency_breakdown", + "tool_calls", + } + + +def test_every_check_is_a_published_score_name(): + expected = { + "tool_call": {"function_arguments": {"keywords": ["Net Sales"]}}, + "expected_objects": ["metric/net_sales"], + "expect_widget": False, + "required_tools": ["search_objects"], + "answer_rubric": ["Lists net sales"], + "expected_limit": 20, + "no_raw_ids": True, + } + with patch(f"{_MODULE}.score_run", return_value=JudgeVerdict(passed=True, reasoning="ok")): + run, _ = _run(expected, _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert set(run.checks) == set(SEARCH_CONDITIONAL_SCORES) + assert run.passed + + +def _dispatch(expected_output: Any) -> dict: + item = DatasetItem( + id="i1", dataset_name="d", test_kind="agentic_search", question="q", expected_output=expected_output + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_search_tool") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + return mock_eval.call_args.kwargs + + +@pytest.mark.parametrize( + "expected_output", + [ + {"tool_call": {"function_arguments": {"keywords": ["x"]}}}, + {"tool_call": {"function_arguments": {"keywords": ["x"]}}, "expected_object": ["metric/x"]}, + {"expected_objects": ["metric/x"]}, + {"expected_object": ["metric/x"]}, + ], +) +def test_the_cli_passes_the_item_shape_whole(expected_output: dict): + kwargs = _dispatch(expected_output) + assert kwargs["expected_output"] == expected_output + assert kwargs["expected_tool_call"] is None + + +@pytest.mark.parametrize("expected_output", ["search", ["metric/x"], None]) +def test_the_cli_rejects_an_expected_output_that_is_not_an_object(expected_output: Any): + with pytest.raises(ValueError, match="must be an object"): + _dispatch(expected_output) + + +def test_the_cli_passes_bare_arguments_as_the_tool_call(): + kwargs = _dispatch({"keywords": ["x"], "object_types": ["metric"]}) + assert kwargs["expected_tool_call"] == {"keywords": ["x"], "object_types": ["metric"]} + assert kwargs["expected_output"] is None + + +# --- Langfuse scores ----------------------------------------------------------------------------- + + +class _FakeCtx: + """Records what the deferred Langfuse block writes, without a Langfuse.""" + + def __init__(self) -> None: + self.scores: dict[str, float] = {} + self.strict: dict = {} + self.run_metadata: dict = {} + + def trace(self, _conversation_id: str) -> None: + return None + + @contextmanager + def observe(self, _trace: Any, _run_idx: int, **_kwargs: Any) -> Any: + yield "trace-id" + + def score(self, _tid: Any, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[name] = float(value) + + def quality(self, _tid: Any, *, strict_checks: dict, latency_sec: Any, cost_usd: Any) -> None: + self.strict = strict_checks + + +def _scored(expected_output: dict, verdict: JudgeVerdict | None = None) -> _FakeCtx: + """Run one item with Langfuse on, then execute the deferred block against a fake ctx.""" + captured: dict = {} + + def _capture(_link: Any, _identity: Any, **kwargs: Any) -> None: + captured.update(kwargs) + + with ( + patch(f"{_MODULE}.ChatClient", return_value=_client(_turn(calls=[_NET_SALES_CALL]))), + patch(f"{_MODULE}.LLMJudge"), + patch(f"{_MODULE}.score_run", return_value=verdict), + patch(f"{_MODULE}.submit_trace_scoring", side_effect=_capture), + ): + try: + evaluate_agentic_search_tool( + host=_HOST, + token="tok", + workspace_id="ws", + question="Find Net Sales", + expected_output=expected_output, + langfuse=MagicMock(), + dataset_item_id="ds-1", + ) + except SearchToolAssertionError: + pass # scores are written before the gate raise + ctx = _FakeCtx() + captured["write_scores"](ctx) + return ctx + + +def test_scores_publish_the_checks_but_not_an_ungraded_answer(): + expected = { + "tool_call": {"function_arguments": {"keywords": ["Net Sales"]}}, + "expect_widget": False, + "answer_rubric": ["Lists net sales"], + } + ctx = _scored(expected, JudgeVerdict(passed=False, reasoning="", error="unreadable")) + assert ctx.scores["tool_selection"] == 1.0 + assert ctx.scores["tool_correctness"] == 1.0 + assert ctx.scores["search_widget"] == 1.0 + assert ctx.scores["eval_error"] == 1.0 + assert "search_answer" not in ctx.scores + assert ctx.strict == {"tool_selection": True, "search_widget": True} + + +def test_scores_of_a_report_only_item_without_search_or_arguments(): + ctx = _scored({"search_required": False, "expect_widget": True, "report_only": True}) + assert "tool_correctness" not in ctx.scores + assert ctx.scores["search_widget"] == 0.0 + assert ctx.strict == {"passed": True}