diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index e596c6128..46b458ff2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -11,6 +11,7 @@ from gooddata_eval.core.agentic._gate import DEFAULT_GATE, EvalGate, normalize_gate from gooddata_eval.core.agentic._langfuse import make_langfuse_client +from gooddata_eval.core.agentic._search_expectation import SEARCH_ARGUMENT_KEYS from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.anomaly_detection import evaluate_agentic_anomaly_detection @@ -253,15 +254,18 @@ def _dispatch_agentic( **lf_kw, ) elif kind == "agentic_search": - eo_dict = eo if isinstance(eo, dict) else {} - tool_call = eo_dict.get("tool_call", {}) - expected_args = tool_call.get("function_arguments", eo_dict) + if not isinstance(eo, dict): + raise ValueError(f"agentic_search expected_output must be an object, got {eo!r}") + eo_dict = eo + # Anything but bare search_objects arguments is parsed as an item, so an unknown key fails it. + is_item_shape = "tool_call" in eo_dict or not eo_dict.keys() <= SEARCH_ARGUMENT_KEYS return evaluate_agentic_search_tool( host=host, token=token, workspace_id=workspace_id, question=item.question, - expected_tool_call=expected_args, + expected_tool_call=None if is_item_shape else eo_dict, + expected_output=eo_dict if is_item_shape else None, k=k, gate=gate, agent_id=agent_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_search_expectation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_search_expectation.py new file mode 100644 index 000000000..68086e356 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_search_expectation.py @@ -0,0 +1,301 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""The search-tool item: the expected ``search_objects`` arguments plus the checks an item opts into. + +An item that carries only ``tool_call``, or reaches the runner as the bare arguments, is scored on +``tool_selection`` (gating) and ``tool_correctness`` (informing). Each other key adds one check, and a +key the module does not know is rejected, so a misspelt check cannot read as a pass. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.models import ChatResult, ToolCallEvent + +SEARCH_TOOL = "search_objects" + +# The keys of an item's expected output beside ``tool_call``. Public so a caller holding a raw item can +# tell the options shape from bare arguments. +SEARCH_OPTION_KEYS = frozenset( + { + "expected_objects", + "expect_widget", + "required_tools", + "forbidden_tools", + "search_required", + "answer_rubric", + "expected_limit", + "no_raw_ids", + "setup_turns", + "report_only", + } +) +_KNOWN_KEYS = SEARCH_OPTION_KEYS | {"tool_call"} +# The search_objects parameters, and emit_widget, the old name of user_requested_search. An expected output with +# any other key is read as an item, so a misspelt check key fails it instead of passing as an argument. +SEARCH_ARGUMENT_KEYS = frozenset( + {"keywords", "object_types", "limit", "compatible_with_afm", "kda_eligible", "user_requested_search", "emit_widget"} +) + +_OBJECT_TYPES = frozenset( + { + "metric", + "fact", + "attribute", + "label", + "date_attribute", + "computed_attribute", + "dataset", + "date", + "parameter", + "visualization", + "dashboard", + "document", + "report", + } +) +# The agent's tools take an attribute by either of its ids, and search returns a date attribute with +# ``type`` attribute and ``final_type`` date_attribute, so these spellings name one object. +_ATTRIBUTE_TYPES = frozenset({"attribute", "label", "date_attribute"}) + +# A ``{type/id}`` chip is the UI's own syntax for a named object and renders as its title, also inside +# inline code. An id outside a chip reaches the user as raw text. +_CHIP = re.compile(r"\{[a-z_]+/[^{}\s]+\}") +_UUID = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE) +_TYPED_ID = re.compile(rf"\b(?:{'|'.join(sorted(_OBJECT_TYPES))})/([A-Za-z0-9_](?:[A-Za-z0-9_.-]*[A-Za-z0-9_])?)") +# Prose pairs type words with a slash ("date/time", "label/value pairs"). Only an id with an underscore, +# a dot or a digit, as the workspace ids have, counts. A one-word id such as metric/revenue is not caught. +_ID_MARK = re.compile(r"[_.0-9]") + + +@dataclass(frozen=True) +class SearchExpectation: + """What one item expects. Every field but ``tool_args`` is an opt-in check. + + ``setup_turns`` must only read the workspace (search, describe, an unsaved chart): ``agentic_search`` runs in + the CLI's parallel phase, beside other items of the same workspace. + """ + + tool_args: dict[str, Any] = field(default_factory=dict) + expected_objects: tuple[str, ...] = () + expect_widget: bool | None = None + required_tools: tuple[str, ...] = () + forbidden_tools: tuple[str, ...] = () + search_required: bool = True + answer_rubric: tuple[str, ...] = () + expected_limit: int | None = None + no_raw_ids: bool = False + setup_turns: tuple[str, ...] = () + report_only: bool = False + + @property + def has_checks(self) -> bool: + """Whether the item opts into any check beyond ``tool_selection`` and ``tool_correctness``.""" + return bool( + self.expected_objects + or self.expect_widget is not None + or self.required_tools + or self.forbidden_tools + or self.answer_rubric + or self.expected_limit is not None + or self.no_raw_ids + or self.setup_turns + or not self.search_required + ) + + +def _string_list(key: str, value: Any) -> tuple[str, ...]: + # isinstance, not str(entry): an unquoted YAML `no` loads as False and would turn into the text "False". + if not isinstance(value, list) or not value: + raise ValueError(f"{key} must be a non-empty list of strings, got {value!r}") + if any(not isinstance(entry, str) or not entry.strip() for entry in value): + raise ValueError(f"{key} must hold non-empty strings only, got {value!r}") + return tuple(value) + + +def _flag(key: str, value: Any) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{key} must be true or false, got {value!r}") + return value + + +def _object_ref(ref: str) -> tuple[str, str]: + object_type, sep, object_id = ref.partition("/") + if not sep or object_type not in _OBJECT_TYPES or not object_id or object_id != object_id.strip(): + raise ValueError(f"expected_objects entry must be '/' with a known type, got {ref!r}") + return object_type, object_id + + +def _tool_args(tool_call: Any) -> dict[str, Any]: + if not isinstance(tool_call, dict): + raise ValueError(f"tool_call must be an object, got {tool_call!r}") + name = tool_call.get("function_name") + if name is not None and name != SEARCH_TOOL: + raise ValueError(f"tool_call.function_name must be {SEARCH_TOOL!r}, got {name!r}") + return check_arguments(tool_call.get("function_arguments", {})) + + +def check_arguments(args: Any) -> dict[str, Any]: + """The expected ``search_objects`` arguments, whichever way the item gives them. + + Raises: + ValueError: The arguments are not an object, or ``compatible_with_afm`` is not lists of objects. + """ + if not isinstance(args, dict): + raise ValueError(f"tool_call.function_arguments must be an object, got {args!r}") + afm = args.get("compatible_with_afm") + if afm is not None and ( + not isinstance(afm, dict) + or not set(afm) <= {"measures", "attributes"} + or any(value is not None and not isinstance(value, list) for value in afm.values()) + or any(not isinstance(item, dict) for value in afm.values() for item in value or []) + ): + raise ValueError(f"compatible_with_afm must hold lists of measure and attribute objects, got {afm!r}") + return dict(args) + + +def parse_expectation(expected_output: Any) -> SearchExpectation: + """Read an item's whole expected output. + + Raises: + ValueError: The expected output is not an object, names an unknown key, or holds a value the + run could not score. Raised before any request is sent. + """ + if not isinstance(expected_output, dict): + raise ValueError(f"expected_output must be an object, got {expected_output!r}") + unknown = sorted(set(expected_output) - _KNOWN_KEYS) + if unknown: + raise ValueError(f"expected_output has unknown keys {unknown}; known keys are {sorted(_KNOWN_KEYS)}") + raw = expected_output + tool_args = _tool_args(raw["tool_call"]) if "tool_call" in raw else {} + + expected_objects: tuple[str, ...] = () + if "expected_objects" in raw: + expected_objects = _string_list("expected_objects", raw["expected_objects"]) + for ref in expected_objects: + _object_ref(ref) + required = _string_list("required_tools", raw["required_tools"]) if "required_tools" in raw else () + forbidden = _string_list("forbidden_tools", raw["forbidden_tools"]) if "forbidden_tools" in raw else () + both = sorted(set(required) & set(forbidden)) + if both: + raise ValueError(f"tools {both} are both required and forbidden") + limit = raw.get("expected_limit") + if "expected_limit" in raw and (not isinstance(limit, int) or isinstance(limit, bool) or limit < 1): + raise ValueError(f"expected_limit must be a positive integer, got {limit!r}") + + spec = SearchExpectation( + tool_args=tool_args, + expected_objects=expected_objects, + expect_widget=_flag("expect_widget", raw["expect_widget"]) if "expect_widget" in raw else None, + required_tools=required, + forbidden_tools=forbidden, + search_required=_flag("search_required", raw["search_required"]) if "search_required" in raw else True, + answer_rubric=_string_list("answer_rubric", raw["answer_rubric"]) if "answer_rubric" in raw else (), + expected_limit=limit, + no_raw_ids=_flag("no_raw_ids", raw["no_raw_ids"]) if "no_raw_ids" in raw else False, + setup_turns=_string_list("setup_turns", raw["setup_turns"]) if "setup_turns" in raw else (), + report_only=_flag("report_only", raw["report_only"]) if "report_only" in raw else False, + ) + if "tool_call" not in raw and not spec.has_checks: + raise ValueError("expected_output asserts nothing: give tool_call or at least one check") + if spec.report_only and not spec.has_checks: + raise ValueError("report_only has no check to report") + return spec + + +def _ref_key(object_type: str, object_id: str) -> tuple[str, str]: + return ("attribute" if object_type in _ATTRIBUTE_TYPES else object_type, object_id) + + +def call_args(tc: ToolCallEvent) -> dict[str, Any]: + """The call's arguments, or {} when they are missing or not a JSON object.""" + args = tc.parsed_arguments() + return args if isinstance(args, dict) else {} + + +def search_calls(events: list[ToolCallEvent]) -> list[ToolCallEvent]: + """The turn's ``search_objects`` calls, in order.""" + return [tc for tc in events if tc.function_name == SEARCH_TOOL] + + +def _objects(container: Any) -> list[dict]: + """The object entries of a search result or widget part; none when its shape is not the expected one.""" + objects = container.get("objects") if isinstance(container, dict) else None + return [obj for obj in objects if isinstance(obj, dict)] if isinstance(objects, list) else [] + + +def returned_refs(result: ChatResult) -> list[str]: + """Every object the turn's searches returned, as ``type/id``, first sighting first. + + Read from the ``search_objects`` results and from the ``searchResults`` widget parts: a search + the agent runs for itself shows no widget, so its result is the only place its objects appear. + """ + refs: dict[tuple[str, str], str] = {} + + def _add(object_type: Any, object_id: Any) -> None: + if isinstance(object_type, str) and isinstance(object_id, str) and object_type and object_id: + refs.setdefault(_ref_key(object_type, object_id), f"{object_type}/{object_id}") + + for tc in search_calls(result.tool_call_events or []): + for obj in _objects(tc.parsed_result()): + _add(obj.get("final_type") or obj.get("type"), obj.get("id")) + for part in result.search_results: + for obj in _objects(part): + _add(obj.get("type"), obj.get("id")) + return list(refs.values()) + + +def _key_of(ref: str) -> tuple[str, str]: + object_type, _, object_id = ref.partition("/") + return _ref_key(object_type, object_id) + + +def missing_objects(expected: tuple[str, ...], returned: list[str]) -> list[str]: + """The expected refs no search returned. Every one must be found, not any one.""" + found = {_key_of(ref) for ref in returned} + return [ref for ref in expected if _key_of(ref) not in found] + + +def raw_ids(text: str) -> list[str]: + """The UUIDs and ``type/id`` spellings in ``text`` that sit outside a ``{type/id}`` chip.""" + outside = _CHIP.sub(" ", text) + typed = [m.group(0) for m in _TYPED_ID.finditer(outside) if _ID_MARK.search(m.group(1))] + return _UUID.findall(outside) + typed + + +def deterministic_checks(spec: SearchExpectation, result: ChatResult) -> tuple[dict[str, bool], dict[str, Any]]: + """The checks of one scored turn that need no judge, with the evidence of each failure.""" + checks: dict[str, bool] = {} + details: dict[str, Any] = {} + + def _check(name: str, ok: bool, why: Any) -> None: + checks[name] = ok + if not ok: + details[name] = why + + events = result.tool_call_events or [] + if spec.expected_objects: + returned = returned_refs(result) + missing = missing_objects(spec.expected_objects, returned) + _check("search_expected_objects", not missing, {"missing": missing, "returned": returned[:30]}) + if spec.expect_widget is not None: + parts = len(result.search_results) + _check("search_widget", (parts > 0) == spec.expect_widget, {"expected": spec.expect_widget, "parts": parts}) + if spec.required_tools or spec.forbidden_tools: + called = [tc.function_name for tc in events] + not_called = [name for name in spec.required_tools if name not in called] + forbidden = [name for name in spec.forbidden_tools if name in called] + _check( + "search_routing", + not not_called and not forbidden, + {"not_called": not_called, "forbidden_called": forbidden, "called": called}, + ) + if spec.expected_limit is not None: + limits = [call_args(tc).get("limit") for tc in search_calls(events)] + _check("search_limit", spec.expected_limit in limits, {"expected": spec.expected_limit, "limits": limits}) + if spec.no_raw_ids: + found = raw_ids(result.text_response or "") + _check("search_no_raw_ids", not found, {"raw_ids": found}) + return checks, details diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index e58a08dd3..9c969e018 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -1,21 +1,42 @@ # (C) 2026 GoodData Corporation. All rights reserved. -"""Agentic search-tool evaluation runner (single-turn).""" +"""Agentic search-tool evaluation runner (one scored turn, after any setup turns).""" from __future__ import annotations import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from typing import Any + +import httpx from gooddata_eval.core.agentic._failed_runs import build_failed_runs +from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, EvalGate, - gate_failure_note, gate_passed, log_gate_scores, stamp_gate_metadata, ) -from gooddata_eval.core.agentic._outcome import agentic_detail, agentic_success, raise_agentic_failure +from gooddata_eval.core.agentic._outcome import ( + JudgeErrorOf, + agentic_detail, + agentic_success, + chat_failure_exit, + failure_message_head, + label_runs, + log_eval_error_score, + raise_agentic_failure, +) +from gooddata_eval.core.agentic._search_expectation import ( + SEARCH_TOOL, + SearchExpectation, + call_args, + check_arguments, + deterministic_checks, + parse_expectation, + search_calls, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -25,23 +46,61 @@ submit_trace_scoring, utc_now, ) -from gooddata_eval.core.chat.sse_client import ChatClient +from gooddata_eval.core.chat.render import render_answer_text +from gooddata_eval.core.chat.sse_client import ChatClient, ChatError from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators._llm_judge import JudgeVerdict, LLMJudge, score_run from gooddata_eval.core.langfuse.item_scope import item_scope from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, + ChatResult, + LoopExit, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, ) +_TRANSPORT_ERRORS: tuple[type[BaseException], ...] = (httpx.HTTPError, TimeoutError, ConnectionError) + + +def _judge_provider_errors() -> tuple[type[BaseException], ...]: + """The errors of a judge request that fails at the provider (timeout, outage, rate limit). + + Such a request leaves the run unscored, not failed. ``openai`` is the optional llm-judge extra, so its + errors join only when it is installed. + """ + try: + from openai import OpenAIError # noqa: PLC0415 + except ImportError: + return _TRANSPORT_ERRORS + return (OpenAIError, *_TRANSPORT_ERRORS) + + _DEFAULT_K = 1 +# Every check score an item can publish beside tool_selection and tool_correctness. Each is published only for an +# item that opts into it, and search_answer not for a run the judge could not grade. +SEARCH_CONDITIONAL_SCORES = ( + "search_expected_objects", + "search_widget", + "search_routing", + "search_answer", + "search_limit", + "search_no_raw_ids", +) + +_RUBRIC_STEPS = [ + "EXPECTED OUTPUT lists the criteria, one per line, that the assistant's LAST reply must meet.", + "ACTUAL OUTPUT holds the conversation in order. Earlier turns are context only.", + "A {type/id} chip in a reply is shown to the user as that object's title.", + "Pass only when the last reply meets every criterion. Fail when it breaks any.", +] + def _tool_selection(tool_call_events: list[ToolCallEvent]) -> bool: """Return True if search_objects was called.""" - return any(tc.function_name == "search_objects" for tc in tool_call_events) + return any(tc.function_name == SEARCH_TOOL for tc in tool_call_events) # Arguments that do not determine search quality, so a fixture naming them must not @@ -77,30 +136,54 @@ def _list_matches(expected: list, actual: list, *, loose: bool) -> bool: return all(e in actual for e in expected) +def _afm_matches(expected: dict, actual: Any) -> bool: + """Every expected measure and attribute is in the call's AFM; a null field in the fixture matches anything. + + The agent states ``aggregation: null`` for a metric and may add objects to the AFM, so neither fails it. + """ + if not isinstance(actual, dict): + return False + if any(actual.get(key) is not None and not isinstance(actual.get(key), list) for key in ("measures", "attributes")): + return False + for key in ("measures", "attributes"): + listed = actual.get(key) + actual_items = [item for item in listed if isinstance(item, dict)] if isinstance(listed, list) else [] + for exp in expected.get(key) or []: + if not isinstance(exp, dict): + return False + if not any(all(v is None or act.get(k) == v for k, v in exp.items()) for act in actual_items): + return False + return True + + +def _call_matches(args: dict, expected_tool_call: dict) -> bool: + for key, exp_val in expected_tool_call.items(): + if key in _IGNORED_SEARCH_ARGS: + continue + act_val = args.get(key) + if key == "compatible_with_afm" and isinstance(exp_val, dict): + if not _afm_matches(exp_val, act_val): + return False + elif isinstance(exp_val, list) and isinstance(act_val, list): + if not _list_matches(exp_val, act_val, loose=key == "keywords"): + return False + elif isinstance(exp_val, str) and isinstance(act_val, str): + if not _loosely_contains(exp_val, act_val): + return False + elif exp_val != act_val: + return False + return True + + def _tool_correctness(tool_call_events: list[ToolCallEvent], expected_tool_call: dict) -> bool: - """Return True if the search_objects call arguments match expected. + """Return True if a search_objects call's arguments match expected. List fields (keywords, object_types) use subset matching: every expected value must appear in the actual call and the agent may include extras. ``keywords`` match by - substring; every other list is compared exactly. + substring; every other list is compared exactly. Any call of the turn may match: the + agent often looks the measure up first and asks for compatible attributes after. """ - for tc in tool_call_events: - if tc.function_name == "search_objects": - args = tc.parsed_arguments() or {} - for key, exp_val in expected_tool_call.items(): - if key in _IGNORED_SEARCH_ARGS: - continue - act_val = args.get(key) - if isinstance(exp_val, list) and isinstance(act_val, list): - if not _list_matches(exp_val, act_val, loose=key == "keywords"): - return False - elif isinstance(exp_val, str) and isinstance(act_val, str): - if not _loosely_contains(exp_val, act_val): - return False - elif exp_val != act_val: - return False - return True - return False + return any(_call_matches(call_args(tc), expected_tool_call) for tc in search_calls(tool_call_events)) @dataclass @@ -118,6 +201,21 @@ class SearchResult: # This run's own wall time -- mirrors the single-shot path's best_run_latency_s # (see core/runner.py's _run_one_item). run_latency_s: float = 0.0 + # The item's opt-in checks of this run, and why each failed one failed. + checks: dict[str, bool] = field(default_factory=dict) + check_details: dict[str, Any] = field(default_factory=dict) + # Why the judge could not grade the answer; the run is then labelled ERROR unless another check failed it. + harness_error: str | None = None + # The checks that read False only because of harness_error. Their scores are not published. + harness_checks: tuple[str, ...] = () + # Set when a setup turn failed, so the question was never sent. + exit_reason: LoopExit | None = None + passed: bool = False + + @property + def judge_error(self) -> str | None: + """The name ``build_failed_runs`` reads a run's ungraded verdict under.""" + return self.harness_error @dataclass @@ -130,19 +228,148 @@ class AgenticSearchSummary: best: SearchResult +def _judge_rubric(judge: LLMJudge, rubric: tuple[str, ...], transcript: list[tuple[str, str]]) -> JudgeVerdict: + """Grade the last reply against the rubric, with the turns before it as context.""" + conversation = "\n\n".join(f"{role}: {text}" for role, text in transcript) + criteria = "\n".join(f"- {criterion}" for criterion in rubric) + question = next((text for role, text in reversed(transcript) if role == "User"), "") + return score_run(judge, input=question, expected_output=criteria, actual_output=conversation) + + +def _setup_failed(conversation_id: str, exit_reason: LoopExit, reason: str, latency_s: float) -> SearchResult: + """A run whose setup turn failed: the agent's FAIL, as a chat failure is for every kind. + + Its latency is the setup turns' own: the question was never sent. + """ + return SearchResult( + conversation_id=conversation_id, + tool_selected=False, + tool_correct=False, + tool_call_names=[], + check_details={"setup_turn": reason}, + exit_reason=exit_reason, + run_latency_s=latency_s, + ) + + +def _execute_run( + client: ChatClient, + conversation_id: str, + question: str, + spec: SearchExpectation, + judge: LLMJudge | None, +) -> SearchResult: + """Send the item's setup turns, then the question, and score the question's turn alone.""" + transcript: list[tuple[str, str]] = [] + setup_started = time.monotonic() + for number, turn in enumerate(spec.setup_turns, start=1): + try: + setup = client.send_message(conversation_id, turn) + except ChatError as exc: + reason = f"setup turn {number} failed: {exc}" + return _setup_failed(conversation_id, chat_failure_exit(exc), reason, time.monotonic() - setup_started) + setup_text = render_answer_text(setup) + if not setup_text and not setup.tool_call_events: + reason = f"setup turn {number} got no answer" + return _setup_failed(conversation_id, LoopExit.AGENT_SILENT, reason, time.monotonic() - setup_started) + transcript += [("User", turn), ("Assistant", setup_text)] + + run_started = time.monotonic() + chat_result = client.send_message(conversation_id, question) + run_latency = time.monotonic() - run_started + tcs = chat_result.tool_call_events or [] + selected = _tool_selection(tcs) + run = SearchResult( + conversation_id=conversation_id, + tool_selected=selected, + tool_correct=selected and _tool_correctness(tcs, spec.tool_args), + tool_call_names=[tc.function_name for tc in tcs], + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), + run_latency_s=run_latency, + ) + if spec.has_checks: + run.checks, run.check_details = deterministic_checks(spec, chat_result) + if spec.answer_rubric: + _score_answer(run, spec, judge, transcript, question, chat_result) + run.passed = (selected or not spec.search_required) and (spec.report_only or all(run.checks.values())) + return run + + +def _score_answer( + run: SearchResult, + spec: SearchExpectation, + judge: LLMJudge | None, + transcript: list[tuple[str, str]], + question: str, + chat_result: ChatResult, +) -> None: + """Grade the reply's text against the rubric. A turn without text fails without asking the judge.""" + answer = (chat_result.text_response or "").strip() + if not answer: + run.checks["search_answer"] = False + run.check_details["search_answer"] = "empty answer: the turn has no text" + return + if judge is None: + raise RuntimeError("answer_rubric needs the LLM judge, which was not built") + try: + verdict = _judge_rubric(judge, spec.answer_rubric, [*transcript, ("User", question), ("Assistant", answer)]) + except _judge_provider_errors() as exc: + verdict = JudgeVerdict(passed=False, reasoning="", error=f"{type(exc).__name__}: {exc}") + run.checks["search_answer"] = verdict.passed if verdict.error is None else False + if verdict.error is not None: + reason = f"judge gave no verdict: {verdict.error}" + run.check_details["search_answer"] = f"harness: {reason}" + run.harness_error = reason + run.harness_checks = ("search_answer",) + elif not verdict.passed: + run.check_details["search_answer"] = verdict.reasoning + + +def _open_harness_error(spec: SearchExpectation) -> JudgeErrorOf: + """How labels read a run's harness error: only when nothing but the harness-scored checks failed it. + + A run that also failed a check the harness did score is the agent's FAIL, whatever the judge would have said. + """ + + def _read(run: Any) -> str | None: + if run.harness_error is None: + return None + gate_ok = run.tool_selected or not spec.search_required + failing = {name for name, ok in run.checks.items() if not ok} + return run.harness_error if gate_ok and failing <= set(run.harness_checks) else None + + return _read + + def run_agentic_search_tool( host: str, token: str, workspace_id: str, question: str, - expected_tool_call: dict, + expected_tool_call: dict | None = None, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, user_context: dict | None = None, + expected_output: dict | None = None, ) -> AgenticSearchSummary: - """Run the search-tool agentic evaluation K times (single-turn each).""" + """Run the search-tool agentic evaluation K times. + + ``expected_tool_call`` is the bare ``search_objects`` arguments. ``expected_output`` is the item's whole expected output (``tool_call`` plus the checks of + ``_search_expectation``); when it has no ``tool_call``, ``expected_tool_call`` supplies the arguments. + + Raises: + ValueError: ``expected_output`` or ``expected_tool_call`` is malformed. Raised before any request. + ImportError, OSError: The item has an answer rubric and the LLM judge cannot be built (the llm-judge + extra or OPENAI_API_KEY is missing). Raised before any request. + """ + spec = _expectation(expected_tool_call, expected_output) + # Built before the first request: a missing judge is a setup error to report at once. + judge = LLMJudge(_RUBRIC_STEPS) if spec.answer_rubric else None run_results: list[SearchResult] = [] client = ChatClient( @@ -156,24 +383,7 @@ def run_agentic_search_tool( try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - run_started = time.monotonic() - chat_result = client.send_message(conv_id_0, question) - tcs = chat_result.tool_call_events or [] - selected = _tool_selection(tcs) - correct = selected and _tool_correctness(tcs, expected_tool_call) - run_results.append( - SearchResult( - conversation_id=conv_id_0, - tool_selected=selected, - tool_correct=correct, - tool_call_names=[tc.function_name for tc in tcs], - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - tool_call_events=list(chat_result.tool_call_events or []), - reasoning_step_events=list(chat_result.reasoning_step_events or []), - run_latency_s=time.monotonic() - run_started, - ) - ) + run_results.append(_execute_run(client, conv_id_0, question, spec, judge)) finally: if initial_conversation_id is None: client.delete_conversation(conv_id_0) @@ -181,34 +391,15 @@ def run_agentic_search_tool( for _ in range(1, k): conv_id = client.create_conversation() try: - run_started = time.monotonic() - chat_result = client.send_message(conv_id, question) - tcs = chat_result.tool_call_events or [] - selected = _tool_selection(tcs) - correct = selected and _tool_correctness(tcs, expected_tool_call) - run_results.append( - SearchResult( - conversation_id=conv_id, - tool_selected=selected, - tool_correct=correct, - tool_call_names=[tc.function_name for tc in tcs], - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - tool_call_events=list(chat_result.tool_call_events or []), - reasoning_step_events=list(chat_result.reasoning_step_events or []), - run_latency_s=time.monotonic() - run_started, - ) - ) + run_results.append(_execute_run(client, conv_id, question, spec, judge)) finally: client.delete_conversation(conv_id) finally: client.close() - # Pass requires only tool_selected — tool_correct is a Langfuse quality metric. - # This matches the original Tavern behavior where only missing the tool call failed the test. - pass_at_k = any(r.tool_selected for r in run_results) - pass_power_k = all(r.tool_selected for r in run_results) - best = max(run_results, key=lambda r: (r.tool_correct, r.tool_selected)) + pass_at_k = any(r.passed for r in run_results) + pass_power_k = all(r.passed for r in run_results) + best = max(run_results, key=lambda r: (r.passed, sum(r.checks.values()), r.tool_correct, r.tool_selected)) return AgenticSearchSummary( run_results=run_results, pass_at_k=pass_at_k, @@ -217,23 +408,52 @@ def run_agentic_search_tool( ) +def _expectation(expected_tool_call: dict | None, expected_output: dict | None) -> SearchExpectation: + """The item's expectation from either calling convention. + + Raises: + ValueError: ``expected_output`` or ``expected_tool_call`` is malformed. + """ + if expected_output is None: + return SearchExpectation(tool_args=check_arguments(expected_tool_call or {})) + spec = parse_expectation(expected_output) + if "tool_call" in expected_output or not expected_tool_call: + return spec + return replace(spec, tool_args=check_arguments(expected_tool_call)) + + class SearchToolAssertionError(AgenticAssertionError): """Raised when a search-tool evaluation fails.""" -def _run_detail(run: SearchResult) -> dict: - """The diagnostic fields for ONE run, shared by the best run and every failing one. +def _gating_checks(run: SearchResult, spec: SearchExpectation) -> dict[str, bool]: + """The run's checks that decide it: none for a report-only item, and never one the judge could not grade.""" + if spec.report_only: + return {} + return {name: ok for name, ok in run.checks.items() if name not in run.harness_checks} + - Extracted so a failing run is described by exactly the same keys as the winning run -- - for this kind that means the tools it actually called, which is the whole question when - the run failed because it selected the wrong one. +def _verdict_fields(run: SearchResult, spec: SearchExpectation) -> dict[str, Any]: + """The run's gating verdicts as top-level booleans of its detail, which the report's quality score counts. + + What does not gate sits under ``diagnostics``: the search verdicts of an item that needs no search, and + the checks of a report-only item. """ - return { - "tool_selected": run.tool_selected, - "tool_correct": run.tool_correct, - "tool_call_names": run.tool_call_names, - "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), - } + searched = {"tool_selected": run.tool_selected, "tool_correct": run.tool_correct} + fields: dict[str, Any] = dict(searched) if spec.search_required else {} + fields.update(_gating_checks(run, spec)) + diagnostics: dict[str, bool] = {} if spec.search_required else dict(searched) + if spec.report_only: + diagnostics.update(run.checks) + if diagnostics: + fields["diagnostics"] = diagnostics + if run.check_details: + fields["check_details"] = run.check_details + if run.harness_error: + fields["harness_error"] = run.harness_error + if run.exit_reason is not None: + fields["exit_reason"] = run.exit_reason.value + return fields def evaluate_agentic_search_tool( @@ -241,7 +461,7 @@ def evaluate_agentic_search_tool( token: str, workspace_id: str, question: str, - expected_tool_call: dict, + expected_tool_call: dict | None = None, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, agent_id: str | None = None, @@ -254,6 +474,7 @@ def evaluate_agentic_search_tool( reasoning_effort: ReasoningEffort | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, user_context: dict | None = None, + expected_output: dict | None = None, gate: EvalGate = DEFAULT_GATE, ) -> AgenticEvalOutcome: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure. @@ -261,7 +482,12 @@ def evaluate_agentic_search_tool( Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an AgenticEvalOutcome on success; on failure the same three values are attached to the raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. + ``expected_tool_call`` and ``expected_output`` are read as in ``run_agentic_search_tool``. + + Raises: + ValueError: ``expected_output`` or ``expected_tool_call`` is malformed. Raised before any request. """ + spec = _expectation(expected_tool_call, expected_output) identity = RunIdentity( host, token, @@ -285,8 +511,25 @@ def evaluate_agentic_search_tool( reasoning_effort=reasoning_effort, agent_id=agent_id, user_context=user_context, + expected_output=expected_output, ) + # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. + run_labels = label_runs(summary.run_results, passed=lambda r: r.passed, judge_error=_open_harness_error(spec)) + + def _strict(run: SearchResult) -> dict[str, bool]: + strict = {"tool_selection": run.tool_selected} if spec.search_required else {} + strict.update(_gating_checks(run, spec)) + return strict or {"passed": run.passed} + + def _run_detail(run: SearchResult) -> dict: + """The diagnostic fields for ONE run, shared by the best run and every failing one.""" + return { + **_verdict_fields(run, spec), + "tool_call_names": run.tool_call_names, + "latency_breakdown": build_latency_breakdown(run.tool_call_events, run.reasoning_step_events), + } + if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. window_end = utc_now() @@ -299,12 +542,18 @@ def _write_scores(ctx: RunTraceContext) -> None: with ctx.observe( pt, run_idx, conversation_id=run.conversation_id, output={"tool_selection": run.tool_selected} ) as tid: + if (run_error := run_labels.error_label(run_idx)) is not None: + log_eval_error_score(ctx, tid, run_error) ctx.score(tid, name="tool_selection", value=float(run.tool_selected), data_type="BOOLEAN") - ctx.score(tid, name="tool_correctness", value=float(run.tool_correct), data_type="BOOLEAN") + if spec.tool_args: + ctx.score(tid, name="tool_correctness", value=float(run.tool_correct), data_type="BOOLEAN") + for name, ok in run.checks.items(): + if name not in run.harness_checks: + ctx.score(tid, name=name, value=float(ok), data_type="BOOLEAN") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, - strict_checks={"tool_selection": run.tool_selected}, + strict_checks=_strict(run), latency_sec=pt.latency if pt else None, cost_usd=pt.total_cost if pt else None, ) @@ -324,32 +573,50 @@ def _write_scores(ctx: RunTraceContext) -> None: item_input=question, ) - runs_passed = sum(1 for r in summary.run_results if r.tool_selected) + runs_passed = sum(1 for r in summary.run_results if r.passed) runs_effective = len(summary.run_results) + def _failed_checks(run: SearchResult) -> list[dict]: + if run.exit_reason is not None: + return failed_check_records([FailedCheck("setup_turn", actual=run.check_details.get("setup_turn"))]) + checks: list[FailedCheck] = [] + if spec.search_required and not run.tool_selected: + checks.append(FailedCheck("tool_selection", expected=True, actual=run.tool_call_names)) + checks += [ + FailedCheck(name, expected=True, actual=run.check_details.get(name, False)) + for name, ok in {**_gating_checks(run, spec), **{n: run.checks[n] for n in run.harness_checks}}.items() + if not ok + ] + return failed_check_records(checks) + best = summary.best detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, - tool_selected=best.tool_selected, - tool_correct=best.tool_correct, + **_verdict_fields(best, spec), tool_call_names=best.tool_call_names, ) # Same predicate pass@K and runs_passed are taken over, so an item's failed_runs and its # counts cannot disagree about which runs failed. failed_runs = build_failed_runs( summary.run_results, - passed=lambda r: r.tool_selected, + labels=run_labels, detail=_run_detail, + failed_checks=_failed_checks, ) if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): - gate_note = gate_failure_note(gate, runs_passed, runs_effective) + ungraded = sum(1 for r in summary.run_results if _open_harness_error(spec)(r) is not None) + gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs, ungraded) + failing = [record["name"] for record in _failed_checks(best)] + checks_str = f" Failed checks: {', '.join(failing)}; details: {best.check_details}." if failing else "" raise_agentic_failure( SearchToolAssertionError, f"Search tool assertion failed. {gate_note} " f"tool_selected={best.tool_selected}, tool_correct={best.tool_correct}. " - f"Tool calls made: {best.tool_call_names}", + f"Tool calls made: {best.tool_call_names}.{checks_str}", + harness_error=run_labels.harness_error_record, + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, @@ -361,6 +628,7 @@ def _write_scores(ctx: RunTraceContext) -> None: run_latencies_s=[r.run_latency_s for r in summary.run_results], ) return agentic_success( + shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, diff --git a/packages/gooddata-eval/tests/test_agentic_search_tool_checks.py b/packages/gooddata-eval/tests/test_agentic_search_tool_checks.py new file mode 100644 index 000000000..5d2fd8886 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_search_tool_checks.py @@ -0,0 +1,676 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +"""The opt-in search-tool checks. Response shapes are trimmed from runs on the ecommerce_demo fixture.""" + +import json +import sys +from contextlib import contextmanager +from dataclasses import fields +from typing import Any +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.core.agentic._search_expectation import ( + SEARCH_OPTION_KEYS, + SearchExpectation, + deterministic_checks, + parse_expectation, + raw_ids, + returned_refs, +) +from gooddata_eval.core.agentic.search_tool import ( + SEARCH_CONDITIONAL_SCORES, + SearchToolAssertionError, + _judge_provider_errors, + _tool_correctness, + evaluate_agentic_search_tool, + run_agentic_search_tool, +) +from gooddata_eval.core.chat.sse_client import ChatError, TurnTimeoutError +from gooddata_eval.core.evaluators._llm_judge import JudgeVerdict +from gooddata_eval.core.models import ChatResult, DatasetItem + +_MODULE = "gooddata_eval.core.agentic.search_tool" +_HOST = "http://host" + + +def _search_call(args: dict, objects: list[dict] | None = None) -> dict: + result = {"objects": objects or [], "returned_count": len(objects or []), "search_results_ref": "search_1"} + return {"functionName": "search_objects", "functionArguments": json.dumps(args), "result": json.dumps(result)} + + +def _obj(object_type: str, object_id: str, final_type: str | None = None) -> dict: + return {"id": object_id, "type": object_type, "final_type": final_type or object_type, "title": object_id} + + +def _widget(*objects: tuple[str, str]) -> dict: + return { + "type": "searchResults", + "objects": [{"id": object_id, "type": object_type, "title": object_id} for object_type, object_id in objects], + "requestedObjectType": objects[0][0] if objects else "metric", + } + + +def _turn( + text: str = "Found 2 results.", + calls: list[dict] | None = None, + widgets: list[dict] | None = None, +) -> ChatResult: + return ChatResult.model_validate( + {"textResponse": text, "toolCallEvents": calls or [], "searchResults": widgets or [], "responseId": "resp"} + ) + + +_NET_SALES_CALL = _search_call( + {"keywords": ["Net Sales"], "object_types": ["metric", "fact"], "limit": 20, "compatible_with_afm": None}, + [_obj("metric", "net_sales"), _obj("metric", "total_sales")], +) +_COMPATIBLE_CALL = _search_call( + { + "keywords": None, + "object_types": ["attribute", "date_attribute"], + "limit": 100, + "compatible_with_afm": { + "measures": [{"id": "net_sales", "type": "metric", "aggregation": None}], + "attributes": [], + }, + }, + [ + _obj("attribute", "customer_created_date.dayOfMonth", "date_attribute"), + _obj("attribute", "product_category"), + ], +) + + +def _client(*turns: Any) -> MagicMock: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = list(turns) + return client + + +def _run(expected_output: dict, *turns: Any, judge: Any = None) -> Any: + client = _client(*turns) + with patch(f"{_MODULE}.ChatClient", return_value=client), patch(f"{_MODULE}.LLMJudge", return_value=judge): + summary = run_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected_output + ) + # The scored turn is the question, sent last in the run's own conversation. + assert client.send_message.call_args.args == ("conv-1", "Find Net Sales") + return summary.best, client + + +def _evaluate(expected_output: dict, *turns: Any) -> Any: + client = _client(*turns) + with patch(f"{_MODULE}.ChatClient", return_value=client): + return evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected_output + ) + + +# --- parsing ------------------------------------------------------------------------------------- + + +def test_an_item_with_only_tool_call_has_no_checks(): + spec = parse_expectation({"tool_call": {"function_name": "search_objects", "function_arguments": {"limit": 10}}}) + assert spec.tool_args == {"limit": 10} + assert spec.has_checks is False + + +@pytest.mark.parametrize( + ("expected_output", "message"), + [ + ("search", "must be an object"), + ({"expect_widgets": True}, "unknown keys"), + ({"expected_objects": ["metrics/net_sales"]}, "known type"), + ({"expected_objects": ["metric/"]}, "known type"), + ({"expected_objects": "metric/net_sales"}, "non-empty list"), + ({"expected_objects": []}, "non-empty list"), + ({"answer_rubric": ["says more exist", " "]}, "non-empty strings"), + ({"setup_turns": [False]}, "non-empty strings"), + ({"expect_widget": "yes"}, "true or false"), + ({"no_raw_ids": 1}, "true or false"), + ({"expected_limit": True}, "positive integer"), + ({"expected_limit": 0}, "positive integer"), + ({"required_tools": ["search_objects"], "forbidden_tools": ["search_objects"]}, "both required and forbidden"), + ({"tool_call": {"function_name": "search_knowledge"}}, "function_name"), + ({"tool_call": "search_objects"}, "tool_call must be an object"), + ({"tool_call": {"function_arguments": ["Net Sales"]}}, "function_arguments must be an object"), + ({"tool_call": {"function_arguments": {"compatible_with_afm": {"measures": 0}}}}, "compatible_with"), + ({"tool_call": {"function_arguments": {"compatible_with_afm": {"metrics": []}}}}, "compatible_with_afm"), + ( + {"tool_call": {"function_arguments": {"compatible_with_afm": {"measures": ["net_sales"]}}}}, + "compatible_with", + ), + ({"tool_call": {"function_arguments": {"compatible_with_afm": {"measures": "net_sales"}}}}, "compatible_with"), + ({}, "asserts nothing"), + ({"tool_call": {"function_arguments": {"keywords": ["x"]}}, "report_only": True}, "no check to report"), + ], +) +def test_a_malformed_item_is_rejected(expected_output: Any, message: str): + with pytest.raises(ValueError, match=message): + parse_expectation(expected_output) + + +@pytest.mark.parametrize("afm", [{"measures": 0}, {"measures": 1}, {"attributes": True}, {"measures": ["net_sales"]}]) +@pytest.mark.parametrize("with_output", [False, True]) +def test_malformed_bare_arguments_fail_before_any_request(afm: dict, with_output: bool): + client = MagicMock() + with patch(f"{_MODULE}.ChatClient", return_value=client), pytest.raises(ValueError, match="compatible_with_afm"): + run_agentic_search_tool( + host=_HOST, + token="tok", + workspace_id="ws", + question="q", + expected_tool_call={"compatible_with_afm": afm}, + expected_output={"expect_widget": True} if with_output else None, + ) + client.send_message.assert_not_called() + + +def test_a_tool_call_without_arguments_is_an_item(): + spec = parse_expectation({"tool_call": {"function_arguments": {}}}) + assert spec.tool_args == {} + assert spec.has_checks is False + + +def test_a_malformed_item_fails_before_any_request(): + client = MagicMock() + with patch(f"{_MODULE}.ChatClient", return_value=client), pytest.raises(ValueError, match="unknown keys"): + evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="q", expected_output={"expected_object": ["x"]} + ) + client.send_message.assert_not_called() + + +def test_every_option_key_is_an_expectation_field(): + assert {f.name for f in fields(SearchExpectation)} >= SEARCH_OPTION_KEYS + + +# --- tool_correctness ---------------------------------------------------------------------------- + + +def test_tool_correctness_accepts_a_later_call(): + events = _turn(calls=[_NET_SALES_CALL, _COMPATIBLE_CALL]).tool_call_events + expected = {"compatible_with_afm": {"measures": [{"id": "net_sales"}]}} + assert _tool_correctness(events, expected) is True + + +def test_tool_correctness_compares_the_afm_loosely(): + events = _turn(calls=[_COMPATIBLE_CALL]).tool_call_events + assert _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales", "aggregation": None}]}}) + assert _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales", "type": "metric"}]}}) + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "sell_through"}]}}) + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales", "type": "fact"}]}}) + + +def test_tool_correctness_fails_bare_afm_ids_instead_of_crashing(): + events = _turn(calls=[_COMPATIBLE_CALL]).tool_call_events + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": ["net_sales"]}}) + + +def test_tool_correctness_fails_an_afm_collection_that_is_not_a_list(): + odd = _search_call({"compatible_with_afm": {"measures": 1, "attributes": None}}) + assert not _tool_correctness( + _turn(calls=[odd]).tool_call_events, {"compatible_with_afm": {"measures": [{"id": "x"}]}} + ) + attrs = _search_call({"compatible_with_afm": {"measures": 1, "attributes": [{"id": "product_category"}]}}) + expected = {"compatible_with_afm": {"attributes": [{"id": "product_category"}]}} + assert not _tool_correctness(_turn(calls=[attrs]).tool_call_events, expected) + + +def test_tool_correctness_reads_null_arguments_as_none(): + null_args = {"functionName": "search_objects", "functionArguments": "null", "result": "null"} + assert _tool_correctness(_turn(calls=[null_args]).tool_call_events, {}) is True + assert not _tool_correctness(_turn(calls=[null_args]).tool_call_events, {"keywords": ["Net Sales"]}) + + +def test_tool_correctness_fails_an_afm_the_agent_never_sent(): + events = _turn(calls=[_NET_SALES_CALL]).tool_call_events + assert not _tool_correctness(events, {"compatible_with_afm": {"measures": [{"id": "net_sales"}]}}) + + +# --- deterministic checks ------------------------------------------------------------------------ + + +def test_expected_objects_are_read_from_an_internal_search_without_widget(): + result = _turn(calls=[_NET_SALES_CALL, _COMPATIBLE_CALL]) + checks, _ = deterministic_checks(parse_expectation({"expected_objects": ["attribute/product_category"]}), result) + assert checks == {"search_expected_objects": True} + + +def test_expected_objects_are_read_from_the_widget(): + result = _turn(widgets=[_widget(("dashboard", "370ec88c-6c35-4d2c-9e2d-3ef7cbd8e2f1"))]) + spec = parse_expectation({"expected_objects": ["dashboard/370ec88c-6c35-4d2c-9e2d-3ef7cbd8e2f1"]}) + assert deterministic_checks(spec, result)[0] == {"search_expected_objects": True} + + +def test_every_expected_object_must_be_returned(): + result = _turn(calls=[_NET_SALES_CALL]) + spec = parse_expectation({"expected_objects": ["metric/net_sales", "metric/total_customers"]}) + checks, details = deterministic_checks(spec, result) + assert checks == {"search_expected_objects": False} + assert details["search_expected_objects"]["missing"] == ["metric/total_customers"] + + +def test_an_attribute_matches_by_any_of_its_spellings(): + result = _turn(calls=[_COMPATIBLE_CALL]) + for ref in ("attribute/customer_created_date.dayOfMonth", "date_attribute/customer_created_date.dayOfMonth"): + assert deterministic_checks(parse_expectation({"expected_objects": [ref]}), result)[0][ + "search_expected_objects" + ] + assert deterministic_checks(parse_expectation({"expected_objects": ["label/product_category"]}), result)[0][ + "search_expected_objects" + ] + + +def test_an_object_type_is_not_ignored(): + result = _turn(calls=[_NET_SALES_CALL]) + assert not deterministic_checks(parse_expectation({"expected_objects": ["fact/net_sales"]}), result)[0][ + "search_expected_objects" + ] + + +def test_returned_objects_of_other_synced_types_are_read(): + date_dataset = _search_call({"keywords": ["date"]}, [_obj("dataset", "dt_order", "date"), _obj("parameter", "p_1")]) + result = _turn(calls=[date_dataset]) + spec = parse_expectation({"expected_objects": ["date/dt_order", "metric/net_sales"]}) + checks, details = deterministic_checks(spec, result) + assert details["search_expected_objects"]["missing"] == ["metric/net_sales"] + assert returned_refs(result) == ["date/dt_order", "parameter/p_1"] + + +def test_a_result_that_is_not_an_object_returns_nothing(): + odd = {"functionName": "search_objects", "functionArguments": "null", "result": '"no objects"'} + result = _turn(calls=[odd]) + assert returned_refs(result) == [] + checks, _ = deterministic_checks(parse_expectation({"expected_objects": ["metric/x"], "expected_limit": 5}), result) + assert checks == {"search_expected_objects": False, "search_limit": False} + + +def test_objects_that_are_not_a_list_return_nothing(): + odd = {"functionName": "search_objects", "functionArguments": "{}", "result": '{"objects": 1}'} + result = _turn(calls=[odd], widgets=[{"type": "searchResults", "objects": 1}]) + assert returned_refs(result) == [] + + +def test_returned_refs_keep_the_first_spelling_and_order(): + result = _turn(calls=[_COMPATIBLE_CALL], widgets=[_widget(("attribute", "customer_created_date.dayOfMonth"))]) + assert returned_refs(result) == ["date_attribute/customer_created_date.dayOfMonth", "attribute/product_category"] + + +@pytest.mark.parametrize( + ("expect", "widgets", "ok"), [(True, 1, True), (True, 0, False), (False, 0, True), (False, 1, False)] +) +def test_widget(expect: bool, widgets: int, ok: bool): + result = _turn(widgets=[_widget(("metric", "net_sales"))] * widgets) + assert deterministic_checks(parse_expectation({"expect_widget": expect}), result)[0] == {"search_widget": ok} + + +def test_routing_needs_every_required_tool_and_no_forbidden_one(): + describe = {"functionName": "get_object_with_children", "functionArguments": "{}"} + result = _turn(calls=[_NET_SALES_CALL, describe]) + ok = parse_expectation({"required_tools": ["search_objects", "get_object_with_children"]}) + assert deterministic_checks(ok, result)[0] == {"search_routing": True} + knowledge = parse_expectation({"forbidden_tools": ["search_knowledge", "get_object_with_children"]}) + checks, details = deterministic_checks(knowledge, result) + assert checks == {"search_routing": False} + assert details["search_routing"]["forbidden_called"] == ["get_object_with_children"] + + +def test_limit_passes_when_any_call_asked_for_it(): + retry = _search_call({"keywords": None, "object_types": ["metric"], "limit": 100}) + first = _search_call({"keywords": None, "object_types": ["metric"], "limit": 20}) + result = _turn(calls=[first, retry]) + assert deterministic_checks(parse_expectation({"expected_limit": 20}), result)[0] == {"search_limit": True} + assert deterministic_checks(parse_expectation({"expected_limit": 50}), result)[0] == {"search_limit": False} + + +@pytest.mark.parametrize( + ("text", "found"), + [ + ("Found 10 eligible metrics. `{metric/orders_all_time}` ignores dashboard filters.", []), + ("It is used in {visualization/03b89e08-ee06-4e1f-88d8-bbd805c691cc}.", []), + ("I searched metric/fact/attribute together.", []), + ("Use a date/time attribute, label/value pairs, the dataset/table or a metric/KPI.", []), + ("Search metric/revenue.", []), + ("The dataset is date/dt_order.", ["date/dt_order"]), + ("See 03b89e08-ee06-4e1f-88d8-bbd805c691cc.", ["03b89e08-ee06-4e1f-88d8-bbd805c691cc"]), + ("The id is metric/net_sales.", ["metric/net_sales"]), + ("Use attribute/customer_created_date.dayOfMonth.", ["attribute/customer_created_date.dayOfMonth"]), + ], +) +def test_raw_ids_allow_chips_only(text: str, found: list[str]): + assert raw_ids(text) == found + + +# --- runs ---------------------------------------------------------------------------------------- + + +def test_setup_turns_go_first_and_only_the_last_turn_is_scored(): + setup = _turn(text="Found 10 results for revenue.", calls=[_search_call({"keywords": ["revenue"]})]) + scored = _turn( + text="Found 5 dashboards.", + calls=[_search_call({"keywords": ["revenue"], "object_types": ["dashboard"]})], + widgets=[_widget(("dashboard", "092929af"))], + ) + expected = { + "tool_call": {"function_arguments": {"object_types": ["dashboard"]}}, + "setup_turns": ["Search for revenue"], + "expect_widget": True, + } + run, client = _run(expected, setup, scored) + assert [c.args[1] for c in client.send_message.call_args_list] == ["Search for revenue", "Find Net Sales"] + assert {c.args[0] for c in client.send_message.call_args_list} == {"conv-1"} + assert run.passed and run.tool_correct + assert len(run.tool_call_events) == 1 + + +def test_a_failed_setup_turn_records_its_own_latency(): + clock = MagicMock() + clock.monotonic.side_effect = [10.0, 12.5] + with patch(f"{_MODULE}.time", clock), pytest.raises(SearchToolAssertionError) as exc_info: + _evaluate({"setup_turns": ["Search for revenue"], "expect_widget": True}, TurnTimeoutError("too slow")) + assert exc_info.value.run_latencies_s == [2.5] + + +@pytest.mark.parametrize( + ("setup", "exit_reason"), + [ + (ChatError("stream broke"), "chat_error"), + (TurnTimeoutError("too slow"), "agent_timeout"), + (_turn(text=""), "agent_silent"), + ], +) +def test_a_failed_setup_turn_is_the_agents_fail(setup: Any, exit_reason: str): + with pytest.raises(SearchToolAssertionError) as exc_info: + _evaluate({"setup_turns": ["Search for revenue"], "expect_widget": True}, setup) + error = exc_info.value + run = error.failed_runs[0] + assert run["outcome"]["status"] == "fail" + assert run["exit_reason"] == exit_reason + assert [check["name"] for check in run["failed_checks"]] == ["setup_turn"] + assert not hasattr(error, "harness_error") + + +def test_search_required_false_passes_without_a_search(): + chart = {"functionName": "create_visualization", "functionArguments": "{}"} + run, _ = _run({"search_required": False, "expect_widget": False}, _turn(text="Here is the chart.", calls=[chart])) + assert run.passed + assert run.tool_selected is False + + +def test_a_failing_check_fails_the_run(): + run, _ = _run({"tool_call": {"function_arguments": {}}, "expect_widget": True}, _turn(calls=[_NET_SALES_CALL])) + assert run.tool_selected + assert not run.passed + + +def test_report_only_checks_do_not_gate(): + expected = {"expect_widget": True, "report_only": True} + outcome = _evaluate(expected, _turn(calls=[_NET_SALES_CALL])) + assert "search_widget" not in outcome.detail + assert outcome.detail["diagnostics"] == {"search_widget": False} + assert outcome.runs_passed == 1 + + +def test_a_run_that_rightly_skips_search_keeps_the_search_verdicts_out_of_quality(): + chart = {"functionName": "create_visualization", "functionArguments": "{}"} + outcome = _evaluate({"search_required": False, "expect_widget": False}, _turn(calls=[chart])) + booleans = {k: v for k, v in outcome.detail.items() if isinstance(v, bool)} + assert booleans == {"search_widget": True} + assert outcome.detail["diagnostics"] == {"tool_selected": False, "tool_correct": False} + + +def test_an_empty_answer_fails_the_rubric_without_asking_the_judge(): + judge = MagicMock() + with patch(f"{_MODULE}.score_run") as score: + run, _ = _run({"answer_rubric": ["Says more exist"]}, _turn(text="", calls=[_NET_SALES_CALL]), judge=judge) + score.assert_not_called() + assert run.checks == {"search_answer": False} + assert run.check_details["search_answer"].startswith("empty answer") + assert run.harness_error is None + + +def test_the_judge_grades_the_last_reply_with_the_setup_as_context(): + expected = {"answer_rubric": ["Names only 1. Overview"], "setup_turns": ["Describe Revenue Trend"]} + with patch(f"{_MODULE}.score_run", return_value=JudgeVerdict(passed=True, reasoning="ok")) as score: + run, _ = _run(expected, _turn(text="It shows revenue."), _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert run.checks == {"search_answer": True} + kwargs = score.call_args.kwargs + assert kwargs["expected_output"] == "- Names only 1. Overview" + assert kwargs["actual_output"].endswith("User: Find Net Sales\n\nAssistant: Found 2 results.") + assert kwargs["actual_output"].startswith("User: Describe Revenue Trend") + + +def test_a_failed_rubric_keeps_the_judges_reasoning(): + verdict = JudgeVerdict(passed=False, reasoning="does not say more exist") + with patch(f"{_MODULE}.score_run", return_value=verdict): + run, _ = _run({"answer_rubric": ["Says more exist"]}, _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert run.checks == {"search_answer": False} + assert run.check_details["search_answer"] == "does not say more exist" + assert run.harness_error is None + + +def test_a_judge_provider_error_leaves_the_answer_ungraded(): + with patch(f"{_MODULE}.score_run", side_effect=TimeoutError("judge timed out")): + run, _ = _run({"answer_rubric": ["Says more exist"]}, _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert run.harness_error == "judge gave no verdict: TimeoutError: judge timed out" + assert run.harness_checks == ("search_answer",) + + +def test_judge_provider_errors_need_no_llm_judge_extra(): + with patch.dict(sys.modules, {"openai": None}): + assert _judge_provider_errors() == (httpx.HTTPError, TimeoutError, ConnectionError) + assert len(_judge_provider_errors()) == 4 + + +def test_a_rubric_without_a_judge_is_a_programming_error(): + with pytest.raises(RuntimeError, match="LLM judge"): + _run({"answer_rubric": ["Says more exist"]}, _turn(calls=[_NET_SALES_CALL]), judge=None) + + +def test_a_judge_without_verdict_labels_the_run_error(): + expected = {"tool_call": {"function_arguments": {"keywords": ["Net Sales"]}}, "answer_rubric": ["Lists net sales"]} + verdict = JudgeVerdict(passed=False, reasoning="", error="unreadable") + client = _client(_turn(calls=[_NET_SALES_CALL])) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.LLMJudge"), + patch(f"{_MODULE}.score_run", return_value=verdict), + pytest.raises(SearchToolAssertionError) as exc_info, + ): + evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected + ) + run = exc_info.value.failed_runs[0] + assert run["outcome"]["status"] == "error" + assert "1 ungraded" in str(exc_info.value) + assert run["error"] == "judge gave no verdict: unreadable" + assert "search_answer" not in exc_info.value.detail + assert exc_info.value.detail["check_details"]["search_answer"].startswith("harness:") + + +def test_a_judge_error_is_not_blamed_on_the_harness_when_another_check_failed(): + expected = {"answer_rubric": ["Lists net sales"], "expect_widget": True} + verdict = JudgeVerdict(passed=False, reasoning="", error="unreadable") + client = _client(_turn(calls=[_NET_SALES_CALL])) + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.LLMJudge"), + patch(f"{_MODULE}.score_run", return_value=verdict), + pytest.raises(SearchToolAssertionError) as exc_info, + ): + evaluate_agentic_search_tool( + host=_HOST, token="tok", workspace_id="ws", question="Find Net Sales", expected_output=expected + ) + assert exc_info.value.failed_runs[0]["outcome"]["status"] == "fail" + + +def test_expected_output_without_tool_call_takes_the_bare_arguments(): + client = _client(_turn(calls=[_NET_SALES_CALL])) + with patch(f"{_MODULE}.ChatClient", return_value=client): + summary = run_agentic_search_tool( + host=_HOST, + token="tok", + workspace_id="ws", + question="Find Net Sales", + expected_tool_call={"object_types": ["dashboard"]}, + expected_output={"expect_widget": False}, + ) + assert summary.best.tool_correct is False + assert summary.best.passed + + +def test_an_item_with_only_tool_call_keeps_the_legacy_detail(): + outcome = _evaluate( + {"tool_call": {"function_name": "search_objects", "function_arguments": {"keywords": ["Net Sales"]}}}, + _turn(calls=[_NET_SALES_CALL]), + ) + assert set(outcome.detail) == { + "tool_selected", + "tool_correct", + "tool_call_names", + "latency_breakdown", + "tool_calls", + } + + +def test_every_check_is_a_published_score_name(): + expected = { + "tool_call": {"function_arguments": {"keywords": ["Net Sales"]}}, + "expected_objects": ["metric/net_sales"], + "expect_widget": False, + "required_tools": ["search_objects"], + "answer_rubric": ["Lists net sales"], + "expected_limit": 20, + "no_raw_ids": True, + } + with patch(f"{_MODULE}.score_run", return_value=JudgeVerdict(passed=True, reasoning="ok")): + run, _ = _run(expected, _turn(calls=[_NET_SALES_CALL]), judge=MagicMock()) + assert set(run.checks) == set(SEARCH_CONDITIONAL_SCORES) + assert run.passed + + +def _dispatch(expected_output: Any) -> dict: + item = DatasetItem( + id="i1", dataset_name="d", test_kind="agentic_search", question="q", expected_output=expected_output + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_search_tool") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + return mock_eval.call_args.kwargs + + +@pytest.mark.parametrize( + "expected_output", + [ + {"tool_call": {"function_arguments": {"keywords": ["x"]}}}, + {"tool_call": {"function_arguments": {"keywords": ["x"]}}, "expected_object": ["metric/x"]}, + {"expected_objects": ["metric/x"]}, + {"expected_object": ["metric/x"]}, + ], +) +def test_the_cli_passes_the_item_shape_whole(expected_output: dict): + kwargs = _dispatch(expected_output) + assert kwargs["expected_output"] == expected_output + assert kwargs["expected_tool_call"] is None + + +@pytest.mark.parametrize("expected_output", ["search", ["metric/x"], None]) +def test_the_cli_rejects_an_expected_output_that_is_not_an_object(expected_output: Any): + with pytest.raises(ValueError, match="must be an object"): + _dispatch(expected_output) + + +def test_the_cli_passes_bare_arguments_as_the_tool_call(): + kwargs = _dispatch({"keywords": ["x"], "object_types": ["metric"]}) + assert kwargs["expected_tool_call"] == {"keywords": ["x"], "object_types": ["metric"]} + assert kwargs["expected_output"] is None + + +# --- Langfuse scores ----------------------------------------------------------------------------- + + +class _FakeCtx: + """Records what the deferred Langfuse block writes, without a Langfuse.""" + + def __init__(self) -> None: + self.scores: dict[str, float] = {} + self.strict: dict = {} + self.run_metadata: dict = {} + + def trace(self, _conversation_id: str) -> None: + return None + + @contextmanager + def observe(self, _trace: Any, _run_idx: int, **_kwargs: Any) -> Any: + yield "trace-id" + + def score(self, _tid: Any, *, name: str, value: Any, data_type: str, comment: str | None = None) -> None: + self.scores[name] = float(value) + + def quality(self, _tid: Any, *, strict_checks: dict, latency_sec: Any, cost_usd: Any) -> None: + self.strict = strict_checks + + +def _scored(expected_output: dict, verdict: JudgeVerdict | None = None) -> _FakeCtx: + """Run one item with Langfuse on, then execute the deferred block against a fake ctx.""" + captured: dict = {} + + def _capture(_link: Any, _identity: Any, **kwargs: Any) -> None: + captured.update(kwargs) + + with ( + patch(f"{_MODULE}.ChatClient", return_value=_client(_turn(calls=[_NET_SALES_CALL]))), + patch(f"{_MODULE}.LLMJudge"), + patch(f"{_MODULE}.score_run", return_value=verdict), + patch(f"{_MODULE}.submit_trace_scoring", side_effect=_capture), + ): + try: + evaluate_agentic_search_tool( + host=_HOST, + token="tok", + workspace_id="ws", + question="Find Net Sales", + expected_output=expected_output, + langfuse=MagicMock(), + dataset_item_id="ds-1", + ) + except SearchToolAssertionError: + pass # scores are written before the gate raise + ctx = _FakeCtx() + captured["write_scores"](ctx) + return ctx + + +def test_scores_publish_the_checks_but_not_an_ungraded_answer(): + expected = { + "tool_call": {"function_arguments": {"keywords": ["Net Sales"]}}, + "expect_widget": False, + "answer_rubric": ["Lists net sales"], + } + ctx = _scored(expected, JudgeVerdict(passed=False, reasoning="", error="unreadable")) + assert ctx.scores["tool_selection"] == 1.0 + assert ctx.scores["tool_correctness"] == 1.0 + assert ctx.scores["search_widget"] == 1.0 + assert ctx.scores["eval_error"] == 1.0 + assert "search_answer" not in ctx.scores + assert ctx.strict == {"tool_selection": True, "search_widget": True} + + +def test_scores_of_a_report_only_item_without_search_or_arguments(): + ctx = _scored({"search_required": False, "expect_widget": True, "report_only": True}) + assert "tool_correctness" not in ctx.scores + assert ctx.scores["search_widget"] == 0.0 + assert ctx.strict == {"passed": True}