diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py new file mode 100644 index 000000000..86a23b375 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py @@ -0,0 +1,511 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Whether a conversation turn's first reply asked the clarifying question its fixture expects.""" + +from __future__ import annotations + +import functools +import re +from collections.abc import Callable +from typing import Any, Literal + +from gooddata_api_client.exceptions import ApiException, NotFoundException +from gooddata_sdk import GoodDataSdk +from gooddata_sdk.utils import load_all_entities +from pydantic import BaseModel, Field, model_validator +from urllib3.exceptions import HTTPError + +from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, retry_transient +from gooddata_eval.core.models import ChatResult + +ClarificationForm = Literal["questionnaire", "text"] + +_REF_RE = re.compile(r"\{([A-Za-z_]+)/([^{}]+)\}") +# The entity kinds a `{type/id}` option reference can name. An attribute and its label are +# spelled either way by the agent, so each accepts the other. A date dataset is a dataset. +_REF_ENTITIES: dict[str, tuple[str, ...]] = { + "metric": ("metrics",), + "fact": ("facts",), + "dataset": ("datasets",), + "date": ("datasets",), + "attribute": ("attributes", "labels"), + "date_attribute": ("attributes", "labels"), + "label": ("labels", "attributes"), + "parameter": ("parameters",), + "dashboard": ("analyticalDashboards",), + "analyticalDashboard": ("analyticalDashboards",), + "visualization": ("visualizationObjects",), + "visualizationObject": ("visualizationObjects",), + "insight": ("visualizationObjects",), +} +# Reference types gen-ai writes that no entity listing holds, so grounding cannot check them. +_UNLISTED_REF_TYPES = frozenset({"computed_attribute", "filter_context", "document", "report"}) +_ENTITY_KINDS = tuple(sorted({kind for kinds in _REF_ENTITIES.values() for kind in kinds})) +_PAGE_SIZE = 1000 +# The EntitiesApi listing of each entity kind. +_LIST_METHODS: dict[str, str] = { + "analyticalDashboards": "get_all_entities_analytical_dashboards", + "attributes": "get_all_entities_attributes", + "datasets": "get_all_entities_datasets", + "facts": "get_all_entities_facts", + "labels": "get_all_entities_labels", + "metrics": "get_all_entities_metrics", + "parameters": "get_all_entities_parameters", + "visualizationObjects": "get_all_entities_visualization_objects", +} +# A shorter skipped option, such as "No" or "Q1", also occurs in text that assumes nothing, so it +# never counts as named. +_MIN_NAMED_OPTION_CHARS = 3 + + +OptionTitleKind = Literal["metric", "attribute", "fact", "dataset", "dashboard", "visualization"] +_TITLE_ENTITIES: dict[str, str] = { + "metric": "metrics", + "attribute": "attributes", + "fact": "facts", + "dataset": "datasets", + "dashboard": "analyticalDashboards", + "visualization": "visualizationObjects", +} +# What may follow a title in an option that adds context, as two charts with one title need. +_TITLE_CONTEXT_SEPARATORS = (" (", " [", " - ", " – ", " — ", ": ", ", ") +_GENERIC_ANSWERS = frozenset( + {"yes", "no", "ok", "okay", "cancel", "confirm", "continue", "proceed", "skip", "not now", "go ahead"} +) +# How an option may spell a chart type whose `local:` id is not a word of its own. +_CHART_TYPE_WORDS: dict[str, tuple[str, ...]] = { + "combo2": ("combo",), + "pushpin": ("geo", "map", "pushpin"), + "heatmap": ("heatmap", "heat map"), + "treemap": ("treemap", "tree map"), + "dependencywheel": ("dependency wheel",), +} + + +class QuestionOptions(BaseModel): + """One question's options: the ``type/id`` references they name, and the options that name none.""" + + refs: list[str] = Field(default_factory=list) + plain: list[str] = Field(default_factory=list) + + +class FirstReply(BaseModel): + """What the turn's first assistant reply asked, read before any simulated-user answer.""" + + asked: bool + form: ClarificationForm | None = None + questions: list[QuestionOptions] = Field(default_factory=list) + + @property + def question_count(self) -> int: + return len(self.questions) + + @property + def option_refs(self) -> list[str]: + return [ref for q in self.questions for ref in q.refs] + + @property + def plain_options(self) -> list[str]: + return [option for q in self.questions for option in q.plain] + + @property + def has_options(self) -> bool: + return any(q.refs or q.plain for q in self.questions) + + +def read_first_reply(chat_result: ChatResult, *, asked_in_text: bool) -> FirstReply: + """The reply's clarifying questions; a ``clarifyingQuestions`` part makes it a questionnaire. + + ``asked_in_text`` is the caller's reading of the prose, for a question without a part. + """ + parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"] + if not parts: + return FirstReply(asked=asked_in_text, form="text" if asked_in_text else None) + questions = [] + for question in (q for p in parts for q in p.get("questions") or []): + options = _options(question) + questions.append( + QuestionOptions( + refs=[f"{kind}/{obj_id}" for text in options for kind, obj_id in _REF_RE.findall(text)], + plain=[text for text in options if not _REF_RE.search(text)], + ) + ) + return FirstReply(asked=True, form="questionnaire", questions=questions) + + +class WorkspaceObjects(BaseModel): + """The workspace's objects as the option checks read them: ``/`` and their titles.""" + + refs: set[str] = Field(default_factory=set) + # `/` -> the title, casefolded with whitespace collapsed. + titles: dict[str, str] = Field(default_factory=dict) + # `visualizationObjects/` -> the chart type, `local:donut` read as `donut`. + chart_types: dict[str, str] = Field(default_factory=dict) + + def titles_of(self, entity_kind: str) -> set[str]: + return {title for ref, title in self.titles.items() if ref.startswith(f"{entity_kind}/")} + + +def _title_key(text: str) -> str: + return " ".join(text.casefold().split()) + + +def _is_transient(exc: Exception) -> bool: + if isinstance(exc, ApiException): + return exc.status in RETRYABLE_STATUS_CODES + return isinstance(exc, HTTPError) + + +def _page_with_retry(list_page: Callable[..., Any], **page: int) -> Any: + return retry_transient(lambda: list_page(**page), is_retryable=_is_transient) + + +def fetch_workspace_objects(sdk: GoodDataSdk, workspace_id: str) -> WorkspaceObjects: + """Every ``/`` and title the workspace holds, inherited objects included. + + A page that fails with a transient error is retried as a chat message is. A kind the server does + not know (404) has no objects. Any other failure of the entity API is raised. + """ + found = WorkspaceObjects() + api = sdk._client.entities_api + for kind in _ENTITY_KINDS: + list_page = functools.partial(getattr(api, _LIST_METHODS[kind]), workspace_id, _check_return_type=False) + try: + data = load_all_entities(functools.partial(_page_with_retry, list_page), page_size=_PAGE_SIZE).data + except NotFoundException: + continue + for obj in data: + if not isinstance(obj, dict) or not obj.get("id"): + continue + ref = f"{kind}/{obj['id']}" + found.refs.add(ref) + raw_attributes = obj.get("attributes") + attributes: dict[str, Any] = raw_attributes if isinstance(raw_attributes, dict) else {} + title = attributes.get("title") + if isinstance(title, str) and title: + found.titles[ref] = _title_key(title) + raw_content = attributes.get("content") + content: dict[str, Any] = raw_content if isinstance(raw_content, dict) else {} + chart_url = str(content.get("visualizationUrl") or "") + if chart_url.startswith("local:"): + found.chart_types[ref] = chart_url.removeprefix("local:") + return found + + +def ungrounded_refs(refs: list[str], known: set[str]) -> list[str]: + """The references that name no object of the workspace, or a kind it has none of. + + A type of ``_UNLISTED_REF_TYPES`` is left unchecked. + """ + missing: list[str] = [] + for ref in refs: + ref_type, _, obj_id = ref.partition("/") + if ref_type in _UNLISTED_REF_TYPES: + continue + kinds = _REF_ENTITIES.get(ref_type, ()) + if not any(f"{kind}/{obj_id}" in known for kind in kinds): + missing.append(ref) + return missing + + +def _is_titled(option: str, titles: set[str]) -> bool: + key = _title_key(option) + return key in titles or any(key.startswith(title + sep) for title in titles for sep in _TITLE_CONTEXT_SEPARATORS) + + +def untitled_options(options: list[str], objects: WorkspaceObjects, kind: OptionTitleKind) -> list[str]: + """The options that are not the title of a ``kind`` object, alone or followed by added context.""" + titles = objects.titles_of(_TITLE_ENTITIES[kind]) + return [option for option in options if not _is_titled(option, titles)] + + +def _off_in_scope( + questions: list[QuestionOptions], off_of: Callable[[QuestionOptions], tuple[int, list[str]]] +) -> list[str]: + """The options an option check fails, read only from the questions the check is about. + + ``off_of`` gives a question's count of options the check reads and the ones it fails. A questionnaire + can ask about more than one thing, such as a metric next to a channel: a question with at least one + option the check accepts is about it, and a question with none is about something else and left out. + When no question is about it, every option the check read fails. + """ + checked = [off_of(q) for q in questions] + about = [off for read, off in checked if len(off) < read] + return [option for off in (about or [off for _, off in checked]) for option in off] + + +def untitled_question_options(reply: FirstReply, objects: WorkspaceObjects, kind: OptionTitleKind) -> list[str]: + """The plain-text options that are not the title of a ``kind`` object, in the questions about that kind.""" + return _off_in_scope(reply.questions, lambda q: (len(q.plain), untitled_options(q.plain, objects, kind))) + + +def _entity_refs(ref: str) -> set[str]: + ref_type, _, obj_id = ref.partition("/") + return {f"{kind}/{obj_id}" for kind in _REF_ENTITIES.get(ref_type, ())} + + +def is_object_ref(ref: str) -> bool: + """Whether ``ref`` is ``type/id`` with a type an option reference can name.""" + ref_type, sep, obj_id = ref.partition("/") + return bool(sep and obj_id) and ref_type in _REF_ENTITIES + + +def _names_chart_type(option: str, chart_type: str) -> bool: + key = _title_key(option) + return any(re.search(rf"\b{re.escape(word)}\b", key) for word in _CHART_TYPE_WORDS.get(chart_type, (chart_type,))) + + +def off_candidate_options(reply: FirstReply, objects: WorkspaceObjects, candidates: list[str]) -> list[str]: + """The options that name none of ``candidates``, in the questions that offer at least one of them. + + An option names a candidate by reference, by its title plus context, or by its chart type when + no other candidate has that type: "Donut chart" tells two same-titled charts apart only then. + A chart type does not count in an option that starts with the title of an object outside the candidates. + """ + wanted = set().union(*(_entity_refs(c) for c in candidates)) + titles = {objects.titles[ref] for ref in wanted if ref in objects.titles} + kinds = {ref.partition("/")[0] for ref in wanted} + other_titles = { + title for ref, title in objects.titles.items() if ref not in wanted and ref.partition("/")[0] in kinds + } - titles + types = [objects.chart_types[ref] for ref in wanted if ref in objects.chart_types] + unique_types = {chart_type for chart_type in types if types.count(chart_type) == 1} + + def names_a_candidate_type(option: str) -> bool: + return any(_names_chart_type(option, t) for t in unique_types) and not _is_titled(option, other_titles) + + def off_of(question: QuestionOptions) -> tuple[int, list[str]]: + off = [ref for ref in question.refs if not (_entity_refs(ref) & wanted)] + off += [ + option for option in question.plain if not _is_titled(option, titles) and not names_a_candidate_type(option) + ] + return len(question.refs) + len(question.plain), off + + return _off_in_scope(reply.questions, off_of) + + +def clarification_mismatches( + reply: FirstReply, + *, + expects_clarification: bool, + expected_form: ClarificationForm | None, + max_questions: int | None, + objects: WorkspaceObjects | None, + grounded: bool, + titled_as: OptionTitleKind | None, + candidates: list[str] | None = None, + reads_questionnaire: bool = False, +) -> list[str]: + """How the first reply differs from the expected clarification; empty when it matches. + + ``grounded`` checks the ``{type/id}`` options of every question; ``titled_as`` checks the plain-text + ones against that kind's titles; ``candidates`` requires every option to name one of those objects. + The last two read only the questions they are about (see ``_off_in_scope``). All three are skipped + when ``objects`` is None: the caller reports the turn ungraded then. + ``reads_questionnaire`` fails a prose question, which has no options for these checks to read. + """ + if not expects_clarification: + return [f"asked a clarifying question ({reply.form}) the request does not need"] if reply.asked else [] + if not reply.asked: + return ["expected a clarifying question, but the first reply asked none"] + out: list[str] = [] + if expected_form is not None and reply.form != expected_form: + out.append(f"clarification form: expected {expected_form}, got {reply.form}") + elif reads_questionnaire and reply.form != "questionnaire": + out.append(f"the turn checks a questionnaire, but the first reply asked as {reply.form}") + if max_questions is not None and reply.question_count > max_questions: + out.append(f"asked {reply.question_count} questions, at most {max_questions} expected") + if objects is not None and grounded: + missing = ungrounded_refs(reply.option_refs, objects.refs) + if missing: + out.append(f"options name objects the workspace does not have: {missing}") + if objects is not None and titled_as is not None: + untitled = untitled_question_options(reply, objects, titled_as) + if untitled: + out.append(f"options are not the title of a {titled_as} of the workspace: {untitled}") + if objects is not None and candidates: + off = off_candidate_options(reply, objects, candidates) + if off: + out.append(f"options name none of {candidates}: {off}") + return out + + +class ClarifyingPick(BaseModel): + """An option the fixture picks in a questionnaire. + + ``option`` is a ``type/id`` reference the option names, or the option's whole text. A list names + the spellings of one option, since the agent writes it as a reference or as plain text. + ``question`` limits the pick to that position; without it, the first question offering the + option takes it. + """ + + option: str | list[str] + question: int | None = Field(default=None, ge=0) + + @property + def spellings(self) -> list[str]: + return [self.option] if isinstance(self.option, str) else list(self.option) + + @property + def label(self) -> str: + return " | ".join(self.spellings) + + @model_validator(mode="after") + def _names_an_option(self) -> ClarifyingPick: + if not self.spellings or not all(s.strip() for s in self.spellings): + raise ValueError("option needs at least one non-empty spelling") + return self + + +class ClarifyingAnswers(BaseModel): + """The ``clarifyingAnswers`` for one questionnaire, and what they left out.""" + + answers: list[dict] = Field(default_factory=list) + skipped: list[dict] = Field(default_factory=list) + matched_picks: list[str] = Field(default_factory=list) + + def render(self) -> str: + """The text gen-ai builds for the model from these answers: every question, a skipped one as Skipped.""" + sections = [(a["questionIndex"], a["questionText"], a["answer"]["optionText"]) for a in self.answers] + sections += [(s["index"], s["text"], "Skipped") for s in self.skipped] + return "\n\n".join(f"**{text}**\n\n{answer}" for _, text, answer in sorted(sections, key=lambda s: s[0])) + + +def _all_questions(chat_result: ChatResult) -> list[dict]: + parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"] + return [q for p in parts for q in p.get("questions") or []] + + +def _questions(chat_result: ChatResult) -> list[dict]: + """The questions of the item the answers name, numbered as gen-ai numbers them: across that item's parts.""" + item_id = chat_result.clarifying_questions_item_id + parts = [ + p + for p in chat_result.unhandled_parts or [] + if p.get("type") == "clarifyingQuestions" and p.get("itemId") in (None, item_id) + ] + return [q for p in parts for q in p.get("questions") or []] + + +def _options(question: dict) -> list[str]: + return [str(o.get("text") or "") for o in (question.get("control") or {}).get("options") or []] + + +def _plain(text: str) -> str: + return " ".join(_REF_RE.sub(" ", text).lower().split()) + + +def _same_object(ref: str, other: str) -> bool: + return ref == other or bool(_entity_refs(ref) & _entity_refs(other)) + + +def option_matches(option_text: str, wanted: str) -> bool: + """Whether an option is the one a pick names, by its ``{type/id}`` reference or its whole text. + + A reference matches the spellings grounding accepts for the same object, such as ``attribute/x`` and ``label/x``. + """ + ref = wanted.strip().strip("{}") + if "/" in ref and any(_same_object(ref, f"{kind}/{obj_id}") for kind, obj_id in _REF_RE.findall(option_text)): + return True + return option_text.strip().lower() == wanted.strip().lower() + + +def build_clarifying_answers(chat_result: ChatResult, picks: list[ClarifyingPick]) -> ClarifyingAnswers: + """Answer the reply's questionnaire with the picks; a question no pick matches is skipped.""" + questions = _questions(chat_result) + out = ClarifyingAnswers() + answered: set[int] = set() + for pick in picks: + for qi, question in enumerate(questions): + if qi in answered or (pick.question is not None and pick.question != qi): + continue + oi = next( + ( + i + for i, text in enumerate(_options(question)) + if any(option_matches(text, spelling) for spelling in pick.spellings) + ), + None, + ) + if oi is None: + continue + answered.add(qi) + out.matched_picks.append(pick.label) + out.answers.append( + { + "questionIndex": qi, + "questionText": question.get("text") or "", + "answer": {"type": "singleChoice", "optionIndex": oi, "optionText": _options(question)[oi]}, + } + ) + break + out.answers.sort(key=lambda a: a["questionIndex"]) + out.skipped = [ + {"index": qi, "text": q.get("text") or "", "options": _options(q)} + for qi, q in enumerate(questions) + if qi not in answered + ] + return out + + +def _names_option(text: str, option: str) -> bool: + """Whether ``text`` names the option: one of its references whole, or its words as whole words.""" + refs_in_text = [f"{kind}/{obj_id}" for kind, obj_id in _REF_RE.findall(text)] + for kind, obj_id in _REF_RE.findall(option): + if any(_same_object(f"{kind}/{obj_id}", ref) for ref in refs_in_text): + return True + label = _plain(option) + return ( + len(label) >= _MIN_NAMED_OPTION_CHARS + and re.search(rf"(? bool: + """Whether ``question`` asks ``skipped`` again: the same text, or the same options that name something. + + Options such as Yes and No fit any question, a confirmation too, so they count only with the same text. + """ + if _plain(question.get("text") or "") == _plain(skipped["text"]): + return True + options = sorted(_options(question)) + specific = any(_plain(option).strip(".!") not in _GENERIC_ANSWERS for option in options) + return specific and options == sorted(skipped["options"]) + + +def after_answers_mismatches( + *, + unmatched_picks: list[str], + skipped: list[dict], + replies_after: list[tuple[str, ChatResult, bool]], + states_skipped_assumption: bool, + skipped_questions_not_reasked: bool, + expects_confirmation_after_answers: bool, +) -> list[str]: + """How the turn after its structured answers differs from the fixture; empty when it matches. + + ``replies_after`` holds each later reply with its kind (``output`` or a ``classify_reply`` kind) and + whether it answers a ``clarifyingAnswers`` message directly. + + ``states_skipped_assumption`` stands in for "the agent says what it assumed": it passes when any reply + after the answers names an option of each skipped question. It cannot tell an assumption from a + mention, and it does not need the assumption in the final reply, since a reply that asks for + confirmation usually states it. + """ + out = [f"no questionnaire offered the picked option {pick!r}" for pick in unmatched_picks] + if states_skipped_assumption: + texts = [reply.text_response or "" for _, reply, _ in replies_after] + out.extend( + f"no reply after the answers names an option of the skipped question {question['text']!r}" + for question in skipped + if not any(_names_option(text, option) for text in texts for option in question["options"]) + ) + if skipped_questions_not_reasked: + for question in skipped: + asked_again = any( + _same_question(later, question) for _, reply, _ in replies_after for later in _all_questions(reply) + ) + if asked_again: + out.append(f"asked the skipped question {question['text']!r} again") + if expects_confirmation_after_answers and any(kind == "output" and follows for kind, _, follows in replies_after): + out.append("acted on the answers in the same reply, without waiting for confirmation") + return out diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py index 55bbd8f89..f14d4c18f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py @@ -68,6 +68,8 @@ # word, so the apostrophes in "don't ... it's" do not mask the text between them. _FORMULA_SPAN_RE = re.compile(r"`+[^`]*`+|\{[^{}]*\}|\"[^\"\n]*\"|(? ReplyKind: return "no_action" +def asks_in_text(chat_result: ChatResult, rendered_text: str) -> bool: + """Whether a reply without the expected output asks the user something, read for the clarification checks. + + ``classify_reply`` reads "Did you mean A or B? Please confirm." as a confirmation, which is right for + what the simulated user says back. Here it is a question: a sentence that ends in a question mark and + is not itself a confirmation phrase asks, whatever confirmation follows it. + """ + kind = classify_reply(chat_result, rendered_text) + if kind != "confirmation" or chat_result.alert_proposals: + return kind == "question" + sentences = _SENTENCE_END_RE.split(rendered_text.strip()) + return any(s.rstrip().endswith("?") and not _CONFIRMATION_RE.search(s) for s in sentences) + + @dataclass class TranscriptEntry: role: Literal["user", "assistant"] diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 3788598a5..e980958bc 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -27,13 +27,28 @@ from gooddata_sdk import GoodDataSdk from pydantic import BaseModel, Field, ValidationError, model_validator +from gooddata_eval.core.agentic._clarification import ( + ClarificationForm, + ClarifyingPick, + FirstReply, + OptionTitleKind, + WorkspaceObjects, + after_answers_mismatches, + build_clarifying_answers, + clarification_mismatches, + fetch_workspace_objects, + is_object_ref, + read_first_reply, +) from gooddata_eval.core.agentic._conversation_context import ( CONFIRMATION_REPLY, NUDGE_MESSAGE, ClarificationJudge, TranscriptEntry, + asks_in_text, classify_reply, clip, + has_structured_question, judge_prompt, render_transcript, reply_from_facts, @@ -108,6 +123,27 @@ class TurnDefinition(BaseModel): and setting it to null clears it. A turn that leaves it out keeps the current one. Its clarification replies carry it too, since gen-ai treats a message without a context as cleared. + + ``expects_clarification`` grades the turn's first reply: true means it must ask, false means + it must not. Left out, nothing is checked. With true, ``expected_clarification_form`` names how + it asks (``questionnaire`` = a ``clarifyingQuestions`` part, ``text`` = prose only), + ``max_clarifying_questions`` caps the questionnaire, and ``clarifying_options_grounded`` + requires every ``{type/id}`` option to name an object of the workspace. ``clarifying_option_titles`` + names an object kind (``visualization``, ``metric``, ...): every plain-text option must be the title + of such an object, alone or followed by context in brackets or after a dash, comma or colon. + ``clarifying_option_objects`` lists the ``type/id`` objects the question is about, such as the + charts sharing the title the user named: every option must name one of them, by reference, by + its title with that context, or by its chart type when no other candidate has that type. + + ``clarifying_answers`` answers the questionnaire of the turn's first reply the way the chat panel + does, with a ``clarifyingAnswers`` message: a question no pick matches is skipped, and an empty + list skips them all. Every other question, a later questionnaire included, gets ``set_answers`` + text, so a follow-up confirmation can be answered. With it, + ``states_skipped_assumption`` requires a reply after the answers to name an option of each skipped + question (a mention, which stands in for the stated assumption), + ``skipped_questions_not_reasked`` forbids a later questionnaire asking one again, and + ``expects_confirmation_after_answers`` requires the agent to stop after each ``clarifyingAnswers`` + message rather than act on it in the same reply: a skip is not an approval. """ turn_id: str @@ -121,6 +157,66 @@ class TurnDefinition(BaseModel): depends_on: list[str] = Field(default_factory=list) set_answers: list[str] = Field(default_factory=list) user_context: dict | None = None + expects_clarification: bool | None = None + expected_clarification_form: ClarificationForm | None = None + max_clarifying_questions: int | None = Field(default=None, ge=1) + clarifying_options_grounded: bool = False + clarifying_option_titles: OptionTitleKind | None = None + clarifying_option_objects: list[str] | None = None + clarifying_answers: list[ClarifyingPick] | None = None + states_skipped_assumption: bool = False + skipped_questions_not_reasked: bool = False + expects_confirmation_after_answers: bool = False + + @property + def reads_questionnaire(self) -> bool: + """Whether a clarification key of the turn reads a ``clarifyingQuestions`` part, so prose cannot satisfy it.""" + return ( + self.max_clarifying_questions is not None + or self.clarifying_options_grounded + or self.clarifying_option_titles is not None + or self.clarifying_option_objects is not None + or self.clarifying_answers is not None + ) + + @model_validator(mode="after") + def _clarification_keys_need_an_expected_question(self) -> TurnDefinition: + # Each of these only reads a question, so without one it would pass vacuously. + stated = [ + name + for name, value in ( + ("expected_clarification_form", self.expected_clarification_form), + ("max_clarifying_questions", self.max_clarifying_questions), + ("clarifying_options_grounded", self.clarifying_options_grounded or None), + ("clarifying_option_titles", self.clarifying_option_titles), + ("clarifying_option_objects", self.clarifying_option_objects), + ("clarifying_answers", self.clarifying_answers), + ) + if value is not None + ] + if stated and self.expects_clarification is not True: + raise ValueError(f"{stated} need expects_clarification: true") + if self.reads_questionnaire and self.expected_clarification_form == "text": + raise ValueError( + "max_clarifying_questions, clarifying_options_grounded, clarifying_option_titles, " + "clarifying_option_objects and clarifying_answers read a questionnaire, not text" + ) + after_answers = [ + name + for name, value in ( + ("states_skipped_assumption", self.states_skipped_assumption), + ("skipped_questions_not_reasked", self.skipped_questions_not_reasked), + ("expects_confirmation_after_answers", self.expects_confirmation_after_answers), + ) + if value + ] + if self.clarifying_option_objects is not None: + bad = [ref for ref in self.clarifying_option_objects if not is_object_ref(ref)] + if not self.clarifying_option_objects or bad: + raise ValueError(f"clarifying_option_objects needs type/id references, got {bad or 'none'}") + if after_answers and self.clarifying_answers is None: + raise ValueError(f"{after_answers} read the turn after its clarifying_answers; set clarifying_answers") + return self class ConversationFixture(BaseModel): @@ -196,6 +292,11 @@ class TurnResult(BaseModel): mismatches: list[str] = Field(default_factory=list) clarifications: list[ClarificationRecord] = Field(default_factory=list) depends_on: list[str] = Field(default_factory=list) + # None when the fixture states no expects_clarification, or the check could not run. + clarification_correct: bool | None = None + clarification_mismatches: list[str] = Field(default_factory=list) + # Why a stated clarification check could not run, so an ungraded turn does not read as a pass. + clarification_ungraded: str | None = None @property def skill_success(self) -> bool: @@ -211,7 +312,13 @@ def stalled(self) -> bool: @property def context_success(self) -> bool: - return self.skill_success and self.output_correct is not False and not self.lost_context and not self.stalled + return ( + self.skill_success + and self.output_correct is not False + and self.clarification_correct is not False + and not self.lost_context + and not self.stalled + ) def failure_reasons(self) -> list[str]: """Why ``context_success`` is False, in reading order; empty when it is True.""" @@ -224,6 +331,8 @@ def failure_reasons(self) -> list[str]: reasons.append(f"skill {self.expected_skill!r} not active") if self.output_correct is False: reasons.extend(self.mismatches or ["output does not match the expected output"]) + if self.clarification_correct is False: + reasons.extend(self.clarification_mismatches) if self.lost_context: reasons.append("asked for information the conversation already established") if self.stalled: @@ -258,6 +367,13 @@ def detail(self) -> dict: out = self.model_dump(include=self._DETAIL_FIELDS) out["context_success"] = self.context_success out["failure_reasons"] = self.failure_reasons() + # Only when the fixture states the check, so an item without one reports what it always did. + if self.clarification_correct is not None: + out["clarification_correct"] = self.clarification_correct + out["clarification_mismatches"] = list(self.clarification_mismatches) + elif self.clarification_ungraded is not None: + out["clarification_correct"] = None + out["clarification_ungraded"] = self.clarification_ungraded return out @@ -669,6 +785,10 @@ def __init__(self) -> None: self.updated_metrics: list[str] = [] self.alerts: list[str] = [] + def state(self) -> tuple[tuple[str, ...], ...]: + """What the conversation has written so far; a change means a workspace listing is out of date.""" + return tuple(self.metrics), tuple(self.updated_metrics), tuple(self.alerts) + def record(self, tool_call_events: list[ToolCallEvent]) -> None: for metric_id, created_new in _metric_creations(tool_call_events): if created_new: @@ -733,8 +853,17 @@ def run_agentic_conversation( ``user_context`` is the context attached when the conversation starts; a turn's own ``user_context`` replaces it from that turn on. + + A fixture with ``expects_clarification`` needs ``context`` mode: legacy mode neither sends + ``clarifying_answers`` nor lets the clarification checks fail the item, so it raises ``ValueError``. """ resolved_mode = resolve_conversation_mode(mode) + graded = [t.turn_id for t in fixture.turns if t.expects_clarification is not None] + if resolved_mode == "legacy" and graded: + raise ValueError( + f"turns {graded} set expects_clarification, which only context mode grades; " + f"pass mode='context' or set {CONVERSATION_MODE_ENV_VAR}=context" + ) if fresh_conversation_per_turn and initial_conversation_id is not None: raise ValueError("fresh_conversation_per_turn needs conversations this run creates itself") judge = clarification_judge if clarification_judge is not None else ClarificationJudge() @@ -786,6 +915,9 @@ def run_agentic_conversation( run_started = time.monotonic() conversation_swap_s = 0.0 # fresh_conversation_per_turn's delete/create, which is not the run's work + workspace_read_s = 0.0 # the option checks' workspace listing, which is not the run's work either + # The option checks' listing, with the created.state() it was read at; read again once that changes. + workspace_listing: tuple[tuple[tuple[str, ...], ...], WorkspaceObjects] | None = None for turn_index, turn in enumerate(fixture.turns): if fresh_conversation_per_turn and turn_index > 0: swap_started = time.monotonic() @@ -842,6 +974,15 @@ def run_agentic_conversation( final_result: ChatResult | None = None records: list[ClarificationRecord] = [] answers_left = list(turn.set_answers) + first_reply: FirstReply | None = None + # Structured answers: the one to send next, what they skipped, and every reply after them. + pending_answers: tuple[str, list[dict]] | None = None + answers_sent = False + # Why clarifying_answers were not sent, when the loop knows it better than what is left after. + answers_unsent: str | None = None + matched_picks: set[str] = set() + skipped_questions: list[dict] = [] + replies_after_answers: list[tuple[str, ChatResult, bool]] = [] # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop # that simply runs out of range() is labelled correctly with no trailing else. @@ -849,8 +990,19 @@ def run_agentic_conversation( for _iter in range(max_clarification_turns + 1): transcript.append(TranscriptEntry("user", current_message)) incomplete = False + follows_answers = pending_answers is not None + is_first_reply = False try: - chat_result = client.send_message(conversation_id, current_message, user_context=current_context) + if pending_answers is not None: + questions_item_id, answers = pending_answers + pending_answers = None + chat_result = client.send_clarifying_answers( + conversation_id, questions_item_id, answers, user_context=current_context + ) + else: + chat_result = client.send_message( + conversation_id, current_message, user_context=current_context + ) except TurnIncompleteError as exc: chat_result = exc.partial_result or ChatResult() incomplete = True @@ -910,10 +1062,30 @@ def run_agentic_conversation( "assistant", " ".join(p for p in (response_text, summarize_visualizations(chat_result)) if p) ) ) + # A reply the server cut short is a stall the loop nudges past, so the next one is the first reply. + if first_reply is None and not incomplete: + asked_in_text = not _check_output_present(resolved_turn, chat_result) and asks_in_text( + chat_result, response_text + ) + first_reply = read_first_reply(chat_result, asked_in_text=asked_in_text) + is_first_reply = True + # The questionnaire clarifying_answers would answer, when this reply carries it. + answerable = ( + turn.clarifying_answers is not None and is_first_reply and has_structured_question(chat_result) + ) + if answers_sent: + reply_kind = ( + "output" + if _check_output_present(resolved_turn, chat_result) + else classify_reply(chat_result, response_text) + ) + replies_after_answers.append((reply_kind, chat_result, follows_answers)) if _check_output_present(resolved_turn, chat_result): if incomplete: records.append(ClarificationRecord(kind="turn_incomplete", agent_message=clip(response_text))) + if answerable: + answers_unsent = "the first reply delivered the output with its questionnaire, so nothing waited on the answers" turn_exit = LoopExit.SUCCESS break @@ -948,6 +1120,11 @@ def run_agentic_conversation( and any(r.kind in ("no_action", "turn_incomplete") for r in records) ) if clarification_turns >= max_clarification_turns or second_stall: + if answerable: + answers_unsent = ( + f"the clarification budget ({max_clarification_turns}) ended the turn " + "before clarifying_answers were sent" + ) records.append(record) break @@ -957,6 +1134,15 @@ def run_agentic_conversation( reply = CONFIRMATION_REPLY elif kind in ("no_action", "turn_incomplete"): reply = NUDGE_MESSAGE + # Before restating: the fixture expects this question, so a lost_context verdict on it + # must not send the picks to a later questionnaire. + elif answerable and turn.clarifying_answers is not None and chat_result.clarifying_questions_item_id: + built = build_clarifying_answers(chat_result, turn.clarifying_answers) + matched_picks.update(built.matched_picks) + skipped_questions = built.skipped + pending_answers = (chat_result.clarifying_questions_item_id, built.answers) + answers_sent = True + reply = built.render() elif record.lost_context is True and not any(r.lost_context is True for r in records): # Once per turn, so a judge mistake cannot loop the user on restating. reply = reply_restating(render_transcript(transcript[:-1]), response_text) @@ -964,6 +1150,8 @@ def run_agentic_conversation( reply = answers_left.pop(0) else: reply = reply_from_facts(render_transcript(transcript[:-1]), response_text, turn.set_answers) + if answerable and not answers_sent: + answers_unsent = "the first reply's questionnaire has no itemId to send clarifying_answers to" record.reply = clip(reply) records.append(record) @@ -988,6 +1176,67 @@ def run_agentic_conversation( if final_result and output_present: output_correct, mismatches = _check_output_correct(resolved_turn, final_result, all_tool_calls) + # A turn with no complete reply has nothing to read; its chat error or missing output fails it. + clarification_correct: bool | None = None + clarification_problems: list[str] = [] + clarification_ungraded: str | None = None + if turn.expects_clarification is not None and first_reply is not None: + checks_options = ( + turn.clarifying_options_grounded + or turn.clarifying_option_titles is not None + or turn.clarifying_option_objects is not None + ) + workspace_objects: WorkspaceObjects | None = None + # Only a questionnaire with options has anything to check. The listing is kept for the run + # and read again once a turn has written to the workspace, since an option can name an + # object an earlier turn created. + if checks_options and first_reply.has_options: + written = created.state() + if workspace_listing is None or workspace_listing[0] != written: + read_started = time.monotonic() + try: + workspace_listing = (written, fetch_workspace_objects(sdk, workspace_id)) + except Exception as exc: # any failure leaves the options unchecked, never passed + workspace_listing = None + clarification_ungraded = f"workspace objects not readable, options left unchecked: {exc}" + print(f"[CLARIFICATION] {clarification_ungraded}") + workspace_read_s += time.monotonic() - read_started + if workspace_listing is not None: + workspace_objects = workspace_listing[1] + clarification_problems = clarification_mismatches( + first_reply, + expects_clarification=turn.expects_clarification, + expected_form=turn.expected_clarification_form, + max_questions=turn.max_clarifying_questions, + objects=workspace_objects, + grounded=turn.clarifying_options_grounded, + titled_as=turn.clarifying_option_titles, + candidates=turn.clarifying_option_objects, + reads_questionnaire=turn.reads_questionnaire, + ) + if turn.clarifying_answers is not None and not answers_sent: + # Every check after the answers would pass with nothing to read. + if answers_unsent is None: + answers_unsent = ( + "the first reply asked no questionnaire to answer" + if first_reply.form != "questionnaire" + else "the turn ended before clarifying_answers were sent" + ) + clarification_problems.append(f"clarifying_answers not sent: {answers_unsent}") + if turn.clarifying_answers is not None and answers_sent: + clarification_problems += after_answers_mismatches( + unmatched_picks=[p.label for p in turn.clarifying_answers if p.label not in matched_picks], + skipped=skipped_questions, + replies_after=replies_after_answers, + states_skipped_assumption=turn.states_skipped_assumption, + skipped_questions_not_reasked=turn.skipped_questions_not_reasked, + expects_confirmation_after_answers=turn.expects_confirmation_after_answers, + ) + if clarification_problems: + clarification_correct = False + elif clarification_ungraded is None: + clarification_correct = True + # Capture metric output for $ref resolution in subsequent turns. if final_result and turn.expected_output_type == "metric": metric_data = _extract_metric_result(all_tool_calls) @@ -1012,11 +1261,14 @@ def run_agentic_conversation( mismatches=mismatches, clarifications=records, depends_on=list(turn.depends_on), + clarification_correct=clarification_correct, + clarification_mismatches=clarification_problems, + clarification_ungraded=clarification_ungraded, ) ) # Taken here, not after the finally: its cleanup is not part of the run. - run_latency_s = time.monotonic() - run_started - conversation_swap_s + run_latency_s = time.monotonic() - run_started - conversation_swap_s - workspace_read_s finally: if owns_conversation and conversation_id: client.delete_conversation(conversation_id) @@ -1208,6 +1460,13 @@ def _write_scores(ctx: RunTraceContext) -> None: value=float(tr.skill_success), data_type="BOOLEAN", ) + if tr.clarification_correct is not None: + ctx.score( + tid, + name=f"turn_{tr.turn_id}_clarification_correct", + value=float(tr.clarification_correct), + data_type="BOOLEAN", + ) strict_checks = { "conversation_success": result.conversation_success, "full_skill_coverage": result.full_skill_coverage, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 0753f52b2..ce7ead5c1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -206,6 +206,7 @@ class _SseAccumulator: alert_proposals: list[dict[str, Any]] = field(default_factory=list) search_results: list[dict[str, Any]] = field(default_factory=list) unhandled_parts: list[dict[str, Any]] = field(default_factory=list) + clarifying_questions_item_id: str | None = None tool_call_events: list[dict[str, Any]] = field(default_factory=list) call_id_to_event_index: dict[str, int] = field(default_factory=dict) reasoning_steps: list[dict[str, Any]] = field(default_factory=list) @@ -226,7 +227,8 @@ def _handle_text(content: dict[str, Any], acc: _SseAccumulator) -> None: acc.text_parts.append(text) -def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator) -> None: +def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator, item_id: str | None = None) -> None: + """Sort the item's parts; a ``clarifyingQuestions`` part keeps ``item_id``, since answers number questions per item.""" for part in content.get("parts", []): ptype = part.get("type") if ptype == "text": @@ -247,7 +249,8 @@ def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator) -> None: else: if ptype not in _KNOWN_PART_TYPES: _log.warning("unknown multipart part type %r; captured as an unhandled part", ptype) - acc.unhandled_parts.append(part) + tagged = ptype == "clarifyingQuestions" and item_id + acc.unhandled_parts.append({**part, "itemId": item_id} if tagged else part) def _handle_reasoning(content: dict[str, Any], acc: _SseAccumulator) -> None: @@ -300,6 +303,7 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult: "alertProposals": acc.alert_proposals, "searchResults": acc.search_results, "unhandledParts": acc.unhandled_parts, + "clarifyingQuestionsItemId": acc.clarifying_questions_item_id, "toolCallEvents": acc.tool_call_events, "reasoningStepCount": len(acc.reasoning_steps), "reasoningSteps": [step["summary"] for step in acc.reasoning_steps], @@ -439,7 +443,9 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult: if ctype == "text": _handle_text(content, acc) elif ctype == "multipart": - _handle_multipart(content, acc) + _handle_multipart(content, acc, item_id=item.get("itemId")) + if any(p.get("type") == "clarifyingQuestions" for p in content.get("parts", [])): + acc.clarifying_questions_item_id = item.get("itemId") or acc.clarifying_questions_item_id elif ctype == "reasoning": _handle_reasoning(content, acc) elif ctype == "toolCall": @@ -540,11 +546,31 @@ def delete_conversation(self, conversation_id: str) -> None: def send_message( self, conversation_id: str, question: str, *, user_context: dict[str, Any] | None = None + ) -> ChatResult: + return self._send(conversation_id, {"type": "text", "text": question}, user_context=user_context) + + def send_clarifying_answers( + self, + conversation_id: str, + questions_item_id: str, + answers: list[dict[str, Any]], + *, + user_context: dict[str, Any] | None = None, + ) -> ChatResult: + """Answer a questionnaire the way the chat panel does: positions plus the texts shown. + + A question left out of ``answers`` is skipped; an empty list skips them all. + """ + content = {"type": "clarifyingAnswers", "questionsItemId": questions_item_id, "answers": answers} + return self._send(conversation_id, content, user_context=user_context) + + def _send( + self, conversation_id: str, content: dict[str, Any], *, user_context: dict[str, Any] | None = None ) -> ChatResult: url = f"{self._base}/{conversation_id}/messages" join_headers, joined = self._join_item_trace(conversation_id) headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers} - body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}} + body: dict[str, Any] = {"item": {"role": "user", "content": content}} options: dict[str, str] = {} if self._reasoning_effort is not None: options["reasoningEffort"] = self._reasoning_effort diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index fb260099f..85f92e74c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -351,6 +351,9 @@ class ChatResult(BaseModel): alert_proposals: list[dict] = Field(default_factory=list, alias="alertProposals") search_results: list[dict] = Field(default_factory=list, alias="searchResults") unhandled_parts: list[dict] = Field(default_factory=list, alias="unhandledParts") + # The itemId of the assistant item holding a clarifyingQuestions part: a clarifyingAnswers + # message names it as questionsItemId. + clarifying_questions_item_id: str | None = Field(default=None, alias="clarifyingQuestionsItemId") tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents") reasoning_step_count: int = Field(default=0, alias="reasoningStepCount") reasoning_steps: list[str] = Field(default_factory=list, alias="reasoningSteps") diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index 37907e4cc..ca0f2dc2d 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -4,10 +4,24 @@ import sys import types from datetime import date +from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest +from gooddata_api_client.exceptions import ApiException, NotFoundException from gooddata_eval.core.agentic import _conversation_context as ctx_mod +from gooddata_eval.core.agentic._clarification import ( + ClarifyingPick, + QuestionOptions, + WorkspaceObjects, + after_answers_mismatches, + build_clarifying_answers, + fetch_workspace_objects, + off_candidate_options, + read_first_reply, + ungrounded_refs, + untitled_options, +) from gooddata_eval.core.agentic._conversation_context import ( CONFIRMATION_REPLY, NO_ANSWER_REPLY, @@ -15,6 +29,7 @@ ClarificationJudge, ClarificationVerdict, _ClarificationLLM, + asks_in_text, classify_reply, judge_prompt, reply_from_facts, @@ -42,6 +57,7 @@ from gooddata_eval.core.chat.sse_client import ChatError as ChatError_ from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent +from urllib3.exceptions import ProtocolError def _skills_tc(*skills): @@ -1525,7 +1541,17 @@ def _run( ) -> tuple[ConversationResult, MagicMock]: client = MagicMock() client.create_conversation.return_value = "conv-1" - client.send_message.side_effect = replies + # Text messages and clarifying answers take their replies from the one queue, in call order. + queue = iter(replies) + + def _next_reply(*_args: object, **_kwargs: object) -> ChatResult: + reply = next(queue) + if isinstance(reply, BaseException): + raise reply + return reply + + client.send_message.side_effect = _next_reply + client.send_clarifying_answers.side_effect = _next_reply fixture = ConversationFixture(id="c", expected_skills=["visualization"], turns=turns) with ( patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=client) as client_cls, @@ -2269,3 +2295,985 @@ def test_a_fixture_level_user_context_is_rejected() -> None: "turns": [{"turn_id": "t1", "message": "m", "expected_skill": "visualization"}], } ) + + +# Clarification expectations: the first reply of a turn, graded against expects_clarification. +# The questionnaire shape is trimmed from a gen-ai reply on the ecommerce_demo eval workspace. + + +def _questionnaire(*questions: list[str]) -> dict: + return { + "type": "clarifyingQuestions", + "questions": [ + { + "text": f"Question {i}?", + "control": {"type": "singleChoice", "options": [{"text": o} for o in options]}, + } + for i, options in enumerate(questions) + ], + } + + +_RETURNS_CARD = _questionnaire(["{metric/returns}", "{metric/total_returns}"]) +_RETURNS_VIZ = _viz(metric="metric/total_returns", dim="label/product_category") + + +def _card_reply(*questionnaires: dict) -> ChatResult: + return _result(text="I found two return measures.", tools=[_SKILLS], unhandledParts=list(questionnaires)) + + +def _clarifying_turn(**kw: object) -> TurnDefinition: + return _viz_turn( + "t1", + expected=_RETURNS_VIZ, + message="Show me returns by product category.", + set_answers=["Use Total Returns."], + **kw, + ) + + +def test_a_turn_without_expects_clarification_is_not_graded_on_its_question() -> None: + result, _ = _run([_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [_clarifying_turn()]) + turn = result.turn_results[0] + assert turn.clarification_correct is None + assert turn.context_success is True + + +def test_an_expected_questionnaire_that_is_asked_passes() -> None: + turn_def = _clarifying_turn( + expects_clarification=True, expected_clarification_form="questionnaire", max_clarifying_questions=4 + ) + result, client = _run([_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [turn_def]) + turn = result.turn_results[0] + assert turn.clarification_correct is True + assert turn.clarification_mismatches == [] + assert turn.context_success is True + assert client.send_message.call_args_list[1].args[1] == "Use Total Returns." + assert turn.detail()["clarification_correct"] is True + + +def test_an_expected_question_that_is_not_asked_fails_the_turn() -> None: + result, _ = _run([_result(viz=_RETURNS_VIZ, tools=[_SKILLS])], [_clarifying_turn(expects_clarification=True)]) + turn = result.turn_results[0] + assert turn.output_correct is True + assert turn.clarification_correct is False + assert turn.context_success is False + assert "expected a clarifying question, but the first reply asked none" in turn.failure_reasons() + + +def test_a_question_the_request_does_not_need_fails_the_turn() -> None: + result, _ = _run( + [_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [_clarifying_turn(expects_clarification=False)] + ) + turn = result.turn_results[0] + assert turn.clarification_correct is False + assert turn.clarification_mismatches == ["asked a clarifying question (questionnaire) the request does not need"] + + +def test_a_confirmation_is_not_a_question_the_request_does_not_need() -> None: + result, _ = _run( + [_result(text="I will chart Total Returns. Please confirm.", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], + [_clarifying_turn(expects_clarification=False)], + ) + assert result.turn_results[0].clarification_correct is True + + +def test_a_prose_question_fails_an_expected_questionnaire() -> None: + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="questionnaire") + result, _ = _run( + [_result(text="Which returns measure do you want?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def] + ) + turn = result.turn_results[0] + assert turn.clarification_correct is False + assert turn.clarification_mismatches == ["clarification form: expected questionnaire, got text"] + + +def test_a_prose_question_passes_an_expected_text_question() -> None: + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="text") + result, _ = _run( + [_result(text="Which time range do you want?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def] + ) + assert result.turn_results[0].clarification_correct is True + + +def test_more_questions_than_allowed_fail_the_turn() -> None: + card = _questionnaire(["{metric/returns}", "{metric/total_returns}"], ["Monthly", "Quarterly"]) + turn_def = _clarifying_turn(expects_clarification=True, max_clarifying_questions=1) + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_mismatches == ["asked 2 questions, at most 1 expected"] + + +def test_an_option_naming_an_object_the_workspace_lacks_fails_grounding() -> None: + card = _questionnaire(["{metric/total_returns}", "{dashboard/2. Sales}", "Plain text option"]) + known = WorkspaceObjects(refs={"metrics/total_returns", "analyticalDashboards/bf439696"}) + turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known) as fetch: + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert fetch.call_count == 1 + assert fetch.call_args.args[1] == "ws" + assert result.turn_results[0].clarification_mismatches == [ + "options name objects the workspace does not have: ['dashboard/2. Sales']" + ] + + +def _created_metric(metric_id: str) -> dict: + return _tool("create_metric", result={"metric_id": metric_id, "created_new": True}) + + +def test_grounded_options_pass_and_the_listing_is_read_once_while_the_run_writes_nothing() -> None: + turns = [ + _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True), + _viz_turn( + "t2", + expected=_RETURNS_VIZ, + message="Again for returns.", + set_answers=["Use Total Returns."], + expects_clarification=True, + clarifying_options_grounded=True, + ), + ] + known = WorkspaceObjects(refs={"metrics/returns", "metrics/total_returns"}) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known) as fetch: + result, _ = _run( + [ + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ), + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ), + ], + turns, + ) + assert fetch.call_count == 1 + assert [t.clarification_correct for t in result.turn_results] == [True, True] + + +def test_the_listing_is_read_again_once_the_run_has_created_an_object() -> None: + turns = [ + _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True), + _viz_turn( + "t2", + expected=_RETURNS_VIZ, + message="Again for returns.", + set_answers=["Use Total Returns."], + expects_clarification=True, + clarifying_options_grounded=True, + ), + ] + known = WorkspaceObjects(refs={"metrics/returns", "metrics/total_returns"}) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known) as fetch: + _run( + [ + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ), + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ, tools=[_created_metric("return_rate")]), + ], + turns, + ) + assert fetch.call_count == 2 + + +def test_the_listing_is_not_read_for_a_first_reply_without_options() -> None: + turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects") as fetch: + result, _ = _run( + [_result(text="Which returns measure do you want?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], + [turn_def], + ) + fetch.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "the turn checks a questionnaire, but the first reply asked as text" + ] + + +@pytest.mark.parametrize( + "error", + [ApiException(status=401, reason="Unauthorized"), ProtocolError("connection reset"), ValueError("x")], + ids=["unauthorized", "connection-reset", "unexpected-body"], +) +def test_an_unreadable_listing_leaves_the_turn_ungraded_rather_than_passed(error: Exception) -> None: + turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", side_effect=error): + result, _ = _run([_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [turn_def]) + turn = result.turn_results[0] + assert turn.clarification_correct is None + assert turn.clarification_ungraded is not None + assert turn.clarification_ungraded.startswith("workspace objects not readable, options left unchecked") + detail = turn.detail() + assert detail["clarification_correct"] is None + assert detail["clarification_ungraded"] == turn.clarification_ungraded + + +def test_an_unreadable_listing_still_fails_a_check_that_does_not_need_it() -> None: + turn_def = _clarifying_turn( + expects_clarification=True, clarifying_options_grounded=True, max_clarifying_questions=1 + ) + card = _questionnaire(["{metric/returns}", "{metric/total_returns}"], ["Monthly", "Quarterly"]) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", side_effect=ValueError("x")): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is False + + +@pytest.mark.parametrize( + ("ref", "known", "grounded"), + [ + ("metric/total_returns", {"metrics/total_returns"}, True), + ("attribute/product_category", {"labels/product_category"}, True), + ("label/date.month", {"attributes/date.month"}, True), + ("dashboard/bf439696", {"analyticalDashboards/bf439696"}, True), + ("metric/return_rate", {"metrics/total_returns"}, False), + ("widget/x", {"metrics/x"}, False), + # gen-ai's own reference types: a date is a date dataset, a date attribute an attribute. + ("date/order_date", {"datasets/order_date"}, True), + ("date_attribute/order_date.month", {"attributes/order_date.month"}, True), + ("parameter/discount", {"metrics/discount"}, False), + ("computed_attribute/sales_bucket", set(), True), + ], +) +def test_ungrounded_refs_accept_each_spelling_of_an_object(ref: str, known: set[str], grounded: bool) -> None: + assert (ungrounded_refs([ref], known) == []) is grounded + + +def test_read_first_reply_keeps_the_options_of_each_question_apart() -> None: + card = _questionnaire(["{metric/returns}", "Returned amount {metric/total_returns}"], ["Monthly", "Weekly"]) + reply = read_first_reply(_result(unhandledParts=[card]), asked_in_text=False) + assert reply.form == "questionnaire" + assert reply.question_count == 2 + assert reply.questions == [ + QuestionOptions(refs=["metric/returns", "metric/total_returns"]), + QuestionOptions(plain=["Monthly", "Weekly"]), + ] + assert reply.option_refs == ["metric/returns", "metric/total_returns"] + assert reply.plain_options == ["Monthly", "Weekly"] + + +@pytest.mark.parametrize( + ("text", "asks"), + [ + ("Did you mean Net Sales or Gross Sales? Please confirm.", True), + ("Which time range do you want?", True), + ("I will chart Total Returns. Please confirm.", False), + ("Should I proceed with Net Sales?", False), + ("Here is the plan. Shall I create it?", False), + ], +) +def test_a_question_followed_by_a_confirmation_phrase_still_asks(text: str, asks: bool) -> None: + assert asks_in_text(_result(text=text), text) is asks + + +def test_a_prose_choice_with_please_confirm_passes_an_expected_text_question() -> None: + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="text") + text = "Did you mean Returns or Total Returns? Please confirm." + result, _ = _run([_result(text=text, tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True + + +def test_read_first_reply_reads_a_reference_type_with_an_underscore() -> None: + card = _questionnaire(["{date_attribute/order_date.month}", "{date_attribute/ship_date.month}"]) + reply = read_first_reply(_result(unhandledParts=[card]), asked_in_text=False) + assert reply.option_refs == ["date_attribute/order_date.month", "date_attribute/ship_date.month"] + assert reply.plain_options == [] + + +def test_clarification_checks_in_legacy_mode_are_refused_before_any_request() -> None: + with pytest.raises(ValueError, match=r"turns \['t1'\] set expects_clarification, which only context mode grades"): + _run([], [_clarifying_turn(expects_clarification=True)], mode="legacy") + + +def test_option_keys_fail_a_question_asked_in_prose() -> None: + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_objects=["metric/total_returns"]) + known = WorkspaceObjects(refs={"metrics/total_returns"}) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known): + result, _ = _run( + [_result(text="Do you mean returns or total returns?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], + [turn_def], + ) + assert result.turn_results[0].clarification_mismatches == [ + "the turn checks a questionnaire, but the first reply asked as text" + ] + + +def test_a_stalled_reply_is_not_the_first_reply_the_question_is_read_from() -> None: + stall = TurnIncompleteError("SSE error 502", status_code=502, reason="max_iterations") + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="questionnaire") + result, _ = _run([stall, _card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + "keys", + [ + {"expected_clarification_form": "questionnaire"}, + {"max_clarifying_questions": 2}, + {"clarifying_options_grounded": True}, + {"expects_clarification": False, "max_clarifying_questions": 2}, + ], +) +def test_a_clarification_key_without_an_expected_question_is_rejected(keys: dict) -> None: + with pytest.raises(ValueError, match="need expects_clarification: true"): + TurnDefinition(turn_id="t1", message="m", expected_skill="visualization", **keys) + + +def test_a_questionnaire_key_on_a_text_question_is_rejected() -> None: + with pytest.raises(ValueError, match="read a questionnaire, not text"): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + expected_clarification_form="text", + max_clarifying_questions=2, + ) + + +def test_a_zero_question_cap_is_rejected() -> None: + with pytest.raises(ValueError): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + max_clarifying_questions=0, + ) + + +def test_the_clarification_verdict_is_written_to_langfuse_only_when_stated() -> None: + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)] + captured: dict = {} + + def _capture(_submit: object, _identity: object, **kwargs: object) -> None: + captured["write_scores"] = kwargs["write_scores"] + + fixture = ConversationFixture( + id="c", expected_skills=["visualization"], turns=[_clarifying_turn(expects_clarification=True)] + ) + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + patch("gooddata_eval.core.agentic.conversation.submit_trace_scoring", _capture), + ): + evaluate_agentic_conversation( + host="h", + token="t", + workspace_id="ws", + fixture=fixture, + langfuse=MagicMock(), + dataset_item_id="item-1", + mode="context", + ) + ctx = MagicMock() + captured["write_scores"](ctx) + scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + assert scores["turn_t1_clarification_correct"] == 1.0 + + +def _entities_sdk(pages: dict[tuple[str, int], list | Exception]) -> MagicMock: + """An sdk whose EntitiesApi serves ``pages``: the objects, or an exception to raise, per (kind, page).""" + sdk = MagicMock() + calls: list[tuple[str, int]] = [] + + def method(name: str) -> object: + def list_page(workspace_id: str, *, page: int, size: int, _check_return_type: bool) -> SimpleNamespace: + assert (workspace_id, _check_return_type) == ("ws", False) + calls.append((name, page)) + served = pages.get((name, page), []) + if isinstance(served, Exception): + raise served + return SimpleNamespace(data=served, included=[]) + + return list_page + + sdk._client.entities_api.configure_mock( + **{ + f"get_all_entities_{kind}": method(kind) + for kind in ( + "analytical_dashboards", + "attributes", + "datasets", + "facts", + "labels", + "metrics", + "parameters", + "visualization_objects", + ) + } + ) + sdk.calls = calls + return sdk + + +def test_fetch_workspace_objects_pages_each_kind_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("gooddata_eval.core.agentic._clarification._PAGE_SIZE", 2) + metric = lambda i: {"id": i, "attributes": {"title": f"Metric {i.upper()}"}} # noqa: E731 + sdk = _entities_sdk( + { + ("metrics", 0): [metric("a"), metric("b"), "junk", {"attributes": {"title": "No id"}}], + ("metrics", 1): [metric("c")], + ("parameters", 0): NotFoundException(status=404, reason="Not Found"), + ("visualization_objects", 0): [ + {"id": "v", "attributes": {"title": "V", "content": {"visualizationUrl": "local:donut"}}} + ], + } + ) + objects = fetch_workspace_objects(sdk, "ws") + assert {r for r in objects.refs if r.startswith("metrics/")} == {"metrics/a", "metrics/b", "metrics/c"} + assert objects.titles_of("metrics") == {"metric a", "metric b", "metric c"} + assert objects.chart_types == {"visualizationObjects/v": "donut"} + assert [p for k, p in sdk.calls if k == "metrics"] == [0, 1] + assert not any(r.startswith("parameters/") for r in objects.refs) + + +def test_fetch_workspace_objects_retries_a_transient_failure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", "0") + sdk = _entities_sdk({("metrics", 0): [{"id": "a", "attributes": {"title": "A"}}]}) + list_metrics = sdk._client.entities_api.get_all_entities_metrics + failures = [ApiException(status=503, reason="Service Unavailable")] + + def flaky(workspace_id: str, **kw: object) -> SimpleNamespace: + if failures: + raise failures.pop() + return list_metrics(workspace_id, **kw) + + sdk._client.entities_api.get_all_entities_metrics = flaky + assert "metrics/a" in fetch_workspace_objects(sdk, "ws").refs + assert not failures + + +def test_fetch_workspace_objects_raises_a_failure_that_is_not_transient() -> None: + sdk = _entities_sdk({("metrics", 0): ApiException(status=401, reason="Unauthorized")}) + with pytest.raises(ApiException): + fetch_workspace_objects(sdk, "ws") + + +# Option checks read only the questions they are about. + + +def test_an_option_check_ignores_a_question_about_something_else() -> None: + # Reproduced on ecommerce_demo with gpt-5.6-luna: one questionnaire asking for the measure and the breakdown. + card = _questionnaire( + ["{metric/total_sales}", "{metric/net_sales}"], + ["{attribute/product_category}", "{attribute/product_brand}"], + ) + known = WorkspaceObjects( + refs={"metrics/total_sales", "metrics/net_sales", "attributes/product_category", "attributes/product_brand"} + ) + turn_def = _clarifying_turn( + expects_clarification=True, + clarifying_options_grounded=True, + clarifying_option_objects=["metric/total_sales", "metric/net_sales"], + ) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + ("questions", "untitled"), + [ + ([["Net Sales", "Total Sales"], ["Email", "Slack"], ["2. Sales", "1. Overview"]], []), + ([["Net Sales", "Gross Margin"], ["Email", "Slack"]], ["Gross Margin"]), + ([["Email", "Slack"]], ["Email", "Slack"]), + ], + ids=["other-questions-left-out", "off-option-in-the-metric-question", "no-question-about-metrics"], +) +def test_a_title_check_reads_the_questions_about_its_kind(questions: list[list[str]], untitled: list[str]) -> None: + titles = WorkspaceObjects(titles={"metrics/net_sales": "net sales", "metrics/total_sales": "total sales"}) + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_titles="metric") + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=titles): + result, _ = _run([_card_reply(_questionnaire(*questions)), _result(viz=_RETURNS_VIZ)], [turn_def]) + expected = [f"options are not the title of a metric of the workspace: {untitled}"] if untitled else [] + assert result.turn_results[0].clarification_mismatches == expected + + +# Structured answers and what the agent does after a skip. + +_SALES_CARD = { + "type": "clarifyingQuestions", + "questions": [ + { + "text": "Which sales measure should the alert monitor?", + "control": { + "type": "singleChoice", + "options": [{"text": "{metric/total_sales}"}, {"text": "{metric/net_sales}"}], + }, + }, + { + "text": "Which notification channel should I use?", + "control": {"type": "singleChoice", "options": [{"text": "HN SMTP"}, {"text": "Default SMTP"}]}, + }, + ], +} +_SALES_VIZ = _viz(metric="metric/net_sales", dim="label/product_category") + + +def _card(part: dict, item_id: str = "q-item", text: str = "Two sales measures match.") -> ChatResult: + return _result(text=text, tools=[_SKILLS], unhandledParts=[part], clarifyingQuestionsItemId=item_id) + + +def _answers_turn(picks: list[dict], **kw: object) -> TurnDefinition: + return _viz_turn( + "t1", + expected=kw.pop("expected", _SALES_VIZ), + message="Show me sales by product category.", + expects_clarification=True, + clarifying_answers=picks, + **kw, + ) + + +def test_picks_are_sent_as_clarifying_answers_and_an_unpicked_question_is_skipped() -> None: + turn_def = _answers_turn([{"option": "metric/net_sales"}]) + result, client = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def]) + client.send_clarifying_answers.assert_called_once_with( + "conv-1", + "q-item", + [ + { + "questionIndex": 0, + "questionText": "Which sales measure should the alert monitor?", + "answer": {"type": "singleChoice", "optionIndex": 1, "optionText": "{metric/net_sales}"}, + } + ], + user_context=None, + ) + turn = result.turn_results[0] + assert turn.clarification_correct is True + assert turn.context_success is True + assert turn.clarifications[0].reply == ( + "**Which sales measure should the alert monitor?** {metric/net_sales} " + "**Which notification channel should I use?** Skipped" + ) + + +def test_a_pick_can_name_the_option_text_and_its_question() -> None: + turn_def = _answers_turn([{"option": "default smtp", "question": 1}], expected=_SALES_VIZ) + _, client = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def]) + answers = client.send_clarifying_answers.call_args.args[2] + assert [(a["questionIndex"], a["answer"]["optionIndex"]) for a in answers] == [(1, 1)] + + +def test_an_empty_pick_list_skips_every_question() -> None: + _, client = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [_answers_turn([])]) + assert client.send_clarifying_answers.call_args.args[2] == [] + + +def test_a_pick_no_questionnaire_offered_fails_the_turn() -> None: + result, _ = _run( + [_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [_answers_turn([{"option": "metric/order_amount"}])] + ) + turn = result.turn_results[0] + assert turn.clarification_correct is False + assert "no questionnaire offered the picked option 'metric/order_amount'" in turn.failure_reasons() + + +def test_a_question_without_a_questionnaire_gets_the_text_answer_and_fails_the_answers_check() -> None: + turn_def = _answers_turn([{"option": "metric/net_sales"}], set_answers=["Use Net Sales."]) + result, client = _run( + [_result(text="Which sales measure do you mean?", tools=[_SKILLS]), _result(viz=_SALES_VIZ)], [turn_def] + ) + client.send_clarifying_answers.assert_not_called() + assert client.send_message.call_args_list[1].args[1] == "Use Net Sales." + assert result.turn_results[0].clarification_mismatches == [ + "the turn checks a questionnaire, but the first reply asked as text", + "clarifying_answers not sent: the first reply asked no questionnaire to answer", + ] + + +def test_answers_not_sent_because_the_first_reply_also_delivered_the_output_say_so() -> None: + reply = _result( + text="Here is net sales.", + viz=_SALES_VIZ, + tools=[_SKILLS], + unhandledParts=[_SALES_CARD], + clarifyingQuestionsItemId="q-item", + ) + result, client = _run([reply], [_answers_turn([{"option": "metric/net_sales"}])]) + client.send_clarifying_answers.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "clarifying_answers not sent: the first reply delivered the output with its questionnaire, " + "so nothing waited on the answers" + ] + + +def test_answers_not_sent_because_the_budget_ended_the_turn_say_so() -> None: + result, client = _run( + [_card(_SALES_CARD)], [_answers_turn([{"option": "metric/net_sales"}])], max_clarification_turns=0 + ) + client.send_clarifying_answers.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "clarifying_answers not sent: the clarification budget (0) ended the turn before clarifying_answers were sent" + ] + + +def test_answers_not_sent_because_the_questionnaire_has_no_item_id_say_so() -> None: + no_id = _result(text="Two sales measures match.", tools=[_SKILLS], unhandledParts=[_SALES_CARD]) + turn_def = _answers_turn([{"option": "metric/net_sales"}], set_answers=["Use Net Sales."]) + result, client = _run([no_id, _result(viz=_SALES_VIZ)], [turn_def]) + client.send_clarifying_answers.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "clarifying_answers not sent: the first reply's questionnaire has no itemId to send clarifying_answers to" + ] + + +def test_a_lost_context_verdict_on_the_expected_questionnaire_does_not_divert_the_answers() -> None: + turn_def = _answers_turn([{"option": "metric/net_sales"}]) + result, client = _run( + [_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def], judge=_FakeJudge(lost_context=True) + ) + assert client.send_clarifying_answers.call_args.args[1] == "q-item" + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + ("final_text", "named"), + [ + ("You skipped the measure, so I used {metric/total_sales}.", True), + ("I used Default SMTP for the channel and net sales for the rest.", False), + ("Here is the chart.", False), + ], +) +def test_the_final_reply_must_name_an_option_of_each_skipped_question(final_text: str, named: bool) -> None: + card = {"type": "clarifyingQuestions", "questions": [_SALES_CARD["questions"][0]]} + turn_def = _answers_turn( + [], states_skipped_assumption=True, expected=_viz(metric="metric/total_sales", dim="label/product_category") + ) + result, _ = _run( + [_card(card), _result(text=final_text, viz=_viz(metric="metric/total_sales", dim="label/product_category"))], + [turn_def], + ) + turn = result.turn_results[0] + assert turn.clarification_correct is named + if not named: + assert turn.clarification_mismatches == [ + "no reply after the answers names an option of the skipped question " + "'Which sales measure should the alert monitor?'" + ] + + +def test_an_assumption_stated_in_the_reply_that_asks_for_confirmation_counts() -> None: + card = {"type": "clarifyingQuestions", "questions": [_SALES_CARD["questions"][0]]} + total_sales = _viz(metric="metric/total_sales", dim="label/product_category") + turn_def = _answers_turn( + [], + set_answers=["Yes, go ahead."], + states_skipped_assumption=True, + expects_confirmation_after_answers=True, + expected=total_sales, + ) + result, _ = _run( + [ + _card(card), + _result(text="I'll assume {metric/total_sales}, OK?", tools=[_SKILLS]), + _result(text="", viz=total_sales), + ], + [turn_def], + ) + assert result.turn_results[0].clarification_correct is True + + +def test_asking_a_skipped_question_again_fails_the_turn() -> None: + reworded = { + "type": "clarifyingQuestions", + "questions": [{"text": "Which measure did you mean?", "control": _SALES_CARD["questions"][0]["control"]}], + } + turn_def = _answers_turn([], skipped_questions_not_reasked=True) + result, client = _run([_card(_SALES_CARD), _card(reworded, item_id="q-2"), _result(viz=_SALES_VIZ)], [turn_def]) + assert client.send_clarifying_answers.call_count == 1 + assert result.turn_results[0].clarification_mismatches == [ + "asked the skipped question 'Which sales measure should the alert monitor?' again" + ] + + +def test_a_new_question_after_a_skip_is_not_a_reasked_one() -> None: + other = { + "type": "clarifyingQuestions", + "questions": [ + {"text": "Which dashboard?", "control": {"options": [{"text": "1. Overview"}, {"text": "2. Sales"}]}} + ], + } + turn_def = _answers_turn([], skipped_questions_not_reasked=True) + result, _ = _run([_card(_SALES_CARD), _card(other, item_id="q-2"), _result(viz=_SALES_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + ("later_text", "reasked"), + [("Should I build the chart?", False), ("Include returns?", True)], +) +def test_yes_no_options_count_as_a_reask_only_with_the_same_text(later_text: str, reasked: bool) -> None: + later = { + "type": "clarifyingQuestions", + "questions": [{"text": later_text, "control": {"options": [{"text": "Yes"}, {"text": "No"}]}}], + } + mismatches = after_answers_mismatches( + unmatched_picks=[], + skipped=[{"index": 0, "text": "Include returns?", "options": ["No", "Yes"]}], + replies_after=[("question", _result(unhandledParts=[later]), True)], + states_skipped_assumption=False, + skipped_questions_not_reasked=True, + expects_confirmation_after_answers=False, + ) + assert mismatches == (["asked the skipped question 'Include returns?' again"] if reasked else []) + + +@pytest.mark.parametrize( + ("options", "final_text", "named"), + [ + (["All", "Top 10"], "Overall revenue grew.", False), + (["All", "Top 10"], "I kept all products.", True), + (["{metric/net_sales}", "{metric/total_sales}"], "I used {metric/net_sales_ytd}.", False), + (["{attribute/product_category}", "{attribute/region}"], "Split by {label/product_category}.", True), + ], +) +def test_a_skipped_option_is_named_only_as_whole_words_or_a_whole_reference( + options: list[str], final_text: str, named: bool +) -> None: + mismatches = after_answers_mismatches( + unmatched_picks=[], + skipped=[{"index": 0, "text": "Which one?", "options": options}], + replies_after=[("output", _result(text=final_text), True)], + states_skipped_assumption=True, + skipped_questions_not_reasked=False, + expects_confirmation_after_answers=False, + ) + assert (mismatches == []) is named + + +def test_a_pick_by_attribute_matches_an_option_written_as_its_label() -> None: + card = _questionnaire(["{label/product_category}", "{label/region}"]) + built = build_clarifying_answers(_card(card), [ClarifyingPick(option="attribute/product_category")]) + assert [(a["questionIndex"], a["answer"]["optionIndex"]) for a in built.answers] == [(0, 0)] + assert built.matched_picks == ["attribute/product_category"] + + +def test_answers_number_the_questions_of_the_item_they_answer_only() -> None: + first = {**_questionnaire(["{metric/returns}", "{metric/total_returns}"]), "itemId": "i-1"} + second = {**_questionnaire(["{metric/net_sales}", "{metric/total_sales}"]), "itemId": "i-2"} + reply = _result(unhandledParts=[first, second], clarifyingQuestionsItemId="i-2") + built = build_clarifying_answers(reply, [ClarifyingPick(option="metric/total_sales")]) + assert [(a["questionIndex"], a["answer"]["optionIndex"]) for a in built.answers] == [(0, 1)] + assert built.skipped == [] + + +def test_acting_on_the_answers_without_a_confirmation_fails_when_one_is_expected() -> None: + turn_def = _answers_turn([], expects_confirmation_after_answers=True) + result, _ = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_mismatches == [ + "acted on the answers in the same reply, without waiting for confirmation" + ] + + +def test_a_confirmation_before_the_output_passes_when_one_is_expected() -> None: + turn_def = _answers_turn([], expects_confirmation_after_answers=True) + result, client = _run( + [_card(_SALES_CARD), _result(text="I will chart Net Sales. Please confirm."), _result(viz=_SALES_VIZ)], + [turn_def], + ) + assert result.turn_results[0].clarification_correct is True + assert client.send_message.call_args_list[-1].args[1] == CONFIRMATION_REPLY + + +@pytest.mark.parametrize( + "keys", + [ + {"states_skipped_assumption": True}, + {"skipped_questions_not_reasked": True}, + {"expects_confirmation_after_answers": True}, + ], +) +def test_an_after_answers_key_without_clarifying_answers_is_rejected(keys: dict) -> None: + with pytest.raises(ValueError, match="set clarifying_answers"): + TurnDefinition(turn_id="t1", message="m", expected_skill="visualization", expects_clarification=True, **keys) + + +def test_clarifying_answers_need_an_expected_questionnaire() -> None: + with pytest.raises(ValueError, match="need expects_clarification: true"): + TurnDefinition(turn_id="t1", message="m", expected_skill="visualization", clarifying_answers=[]) + with pytest.raises(ValueError, match="not text"): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + expected_clarification_form="text", + clarifying_answers=[], + ) + + +def test_rendered_answers_follow_the_text_gen_ai_builds_for_the_model() -> None: + built = build_clarifying_answers(_card(_SALES_CARD), [ClarifyingPick(option="HN SMTP")]) + assert [s["index"] for s in built.skipped] == [0] + assert built.render() == ( + "**Which sales measure should the alert monitor?**\n\nSkipped\n\n" + "**Which notification channel should I use?**\n\nHN SMTP" + ) + + +def test_a_plan_the_agent_waits_on_after_the_answers_counts_as_stopping_for_confirmation() -> None: + turn_def = _answers_turn([], expects_confirmation_after_answers=True) + result, _ = _run( + [ + _card(_SALES_CARD), + _result(text="I'll use Net Sales. The plan: one bar chart by category."), + _result(viz=_SALES_VIZ), + ], + [turn_def], + ) + assert result.turn_results[0].clarification_mismatches == [] + + +def test_a_pick_names_the_spellings_of_one_option() -> None: + plain = { + "type": "clarifyingQuestions", + "questions": [ + {"text": "Which sales measure?", "control": {"options": [{"text": "Total Sales"}, {"text": "Net Sales"}]}} + ], + } + turn_def = _answers_turn([{"option": ["metric/net_sales", "Net Sales"]}]) + result, client = _run([_card(plain), _result(viz=_SALES_VIZ)], [turn_def]) + assert client.send_clarifying_answers.call_args.args[2][0]["answer"]["optionText"] == "Net Sales" + assert result.turn_results[0].clarification_correct is True + + +def test_only_the_first_questionnaire_gets_the_structured_answers() -> None: + confirm = { + "type": "clarifyingQuestions", + "questions": [{"text": "Should I build it?", "control": {"options": [{"text": "Yes"}, {"text": "No"}]}}], + } + turn_def = _answers_turn([], set_answers=["Yes, build it."], expects_confirmation_after_answers=True) + result, client = _run([_card(_SALES_CARD), _card(confirm, item_id="q-2"), _result(viz=_SALES_VIZ)], [turn_def]) + assert client.send_clarifying_answers.call_count == 1 + assert client.send_message.call_args_list[-1].args[1] == "Yes, build it." + assert result.turn_results[0].clarification_correct is True + + +def test_an_empty_option_spelling_is_rejected() -> None: + with pytest.raises(ValueError, match="non-empty spelling"): + ClarifyingPick(option=[]) + + +# Plain-text options checked against the titles of one object kind (search: B15). + +# Two charts share one title, and the second one's description is another chart's title (ecommerce_demo). +_NSB_1, _NSB_2, _SALES_GOAL = ( + "visualizationObjects/643699f3", + "visualizationObjects/68ddf963", + "visualizationObjects/8a07a527", +) +_CHART_TITLES = WorkspaceObjects( + refs={_NSB_1, _NSB_2, _SALES_GOAL, "visualizationObjects/700d", "visualizationObjects/8619", "metrics/net_sales"}, + titles={ + _NSB_1: "net sales breakdown", + _NSB_2: "net sales breakdown", + _SALES_GOAL: "sales goal", + "visualizationObjects/700d": "net sales", + "visualizationObjects/8619": "total sales", + "metrics/net_sales": "net sales", + }, + chart_types={_NSB_1: "donut", _NSB_2: "waterfall", _SALES_GOAL: "bullet", "visualizationObjects/700d": "donut"}, +) + + +@pytest.mark.parametrize( + ("options", "untitled"), + [ + (["Net Sales Breakdown (1. Overview)", "Net Sales Breakdown (2. Sales)"], []), + (["Net Sales Breakdown - by tab", "net sales"], []), + (["Net sales split", "Net Sales by Region chart"], ["Net sales split", "Net Sales by Region chart"]), + ], +) +def test_plain_options_must_be_titles_of_the_named_kind(options: list[str], untitled: list[str]) -> None: + card = { + "type": "clarifyingQuestions", + "questions": [{"text": "Which chart?", "control": {"options": [{"text": o} for o in options]}}], + } + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_titles="visualization") + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=_CHART_TITLES): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + expected = [f"options are not the title of a visualization of the workspace: {untitled}"] if untitled else [] + assert result.turn_results[0].clarification_mismatches == expected + + +def test_title_check_reads_only_options_without_a_reference_and_only_the_named_kind() -> None: + reply = read_first_reply( + _result(unhandledParts=[_questionnaire(["{visualization/abc}", "Total Sales", "Gross Profit Trend"])]), + asked_in_text=False, + ) + assert reply.plain_options == ["Total Sales", "Gross Profit Trend"] + assert untitled_options(reply.plain_options, _CHART_TITLES, "visualization") == ["Gross Profit Trend"] + assert untitled_options(["Total Sales"], _CHART_TITLES, "metric") == ["Total Sales"] + + +def test_a_title_kind_needs_an_expected_questionnaire() -> None: + with pytest.raises(ValueError, match="need expects_clarification: true"): + TurnDefinition(turn_id="t1", message="m", expected_skill="search", clarifying_option_titles="visualization") + with pytest.raises(ValueError): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="search", + expects_clarification=True, + clarifying_option_titles="chart", + ) + + +_NSB_CANDIDATES = ["visualization/643699f3", "visualization/68ddf963"] + + +@pytest.mark.parametrize( + ("options", "off"), + [ + # gen-ai replies to "Describe the Net Sales Breakdown chart." on ecommerce_demo (gpt-5.6-luna). + (["{visualization/643699f3}", "{visualization/68ddf963}"], []), + (["Net Sales Breakdown", "Sales Goal"], ["Sales Goal"]), + (["Net Sales", "Sales Goal"], ["Net Sales", "Sales Goal"]), + (["Net Sales Breakdown (1. Overview)", "Net Sales Breakdown (2. Sales)"], []), + (["{visualization/8a07a527}", "{visualization/643699f3}"], ["visualization/8a07a527"]), + (["Donut chart", "Waterfall chart"], []), + (["The donut one", "Bullet chart"], ["Bullet chart"]), + (["Net Sales (donut)", "Waterfall chart"], ["Net Sales (donut)"]), + ], +) +def test_every_option_must_name_one_of_the_candidate_objects(options: list[str], off: list[str]) -> None: + card = { + "type": "clarifyingQuestions", + "questions": [{"text": "Which chart?", "control": {"options": [{"text": o} for o in options]}}], + } + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_objects=_NSB_CANDIDATES) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=_CHART_TITLES): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + expected = [f"options name none of {_NSB_CANDIDATES}: {off}"] if off else [] + assert result.turn_results[0].clarification_mismatches == expected + + +def test_a_chart_type_names_a_candidate_only_when_no_other_candidate_has_it() -> None: + reply = read_first_reply( + _result(unhandledParts=[_questionnaire(["Donut chart", "Waterfall chart"])]), asked_in_text=False + ) + same_type = _CHART_TITLES.model_copy(update={"chart_types": {_NSB_1: "donut", _NSB_2: "donut"}}) + assert off_candidate_options(reply, same_type, _NSB_CANDIDATES) == ["Donut chart", "Waterfall chart"] + + +def test_candidate_objects_accept_date_references() -> None: + turn = TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + clarifying_option_objects=["date/order_date", "date_attribute/ship_date.month"], + ) + assert turn.clarifying_option_objects == ["date/order_date", "date_attribute/ship_date.month"] + + +@pytest.mark.parametrize("objects", [[], ["643699f3"], ["chart/643699f3"]]) +def test_candidate_objects_need_type_id_references(objects: list[str]) -> None: + with pytest.raises(ValueError, match="clarifying_option_objects needs type/id references"): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="search", + expects_clarification=True, + clarifying_option_objects=objects, + ) diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 510f2b0ca..629585358 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -1422,3 +1422,49 @@ def test_parse_sse_lines_adhoc_fallback_keeps_its_id_over_one_in_the_arguments() result = parse_sse_lines(lines) assert result.created_visualizations is not None assert result.created_visualizations.objects[0].id == "adhoc-visualization-not-persisted" + + +def test_parse_sse_lines_keeps_the_item_id_of_the_questionnaire(): + card = {"type": "clarifyingQuestions", "questions": [{"text": "Which one?", "control": {"options": []}}]} + lines = [ + 'data: {"item": {"itemId": "i-text", "role": "assistant", "content": ' + '{"type": "multipart", "parts": [{"type": "text", "text": "Two measures."}]}}}', + "data: " + + json.dumps( + { + "item": { + "itemId": "i-card", + "role": "assistant", + "content": {"type": "multipart", "parts": [{"type": "text", "text": "Pick one."}, card]}, + } + } + ), + ] + result = parse_sse_lines(lines) + assert result.clarifying_questions_item_id == "i-card" + assert result.unhandled_parts == [{**card, "itemId": "i-card"}] + + +def test_parse_sse_lines_has_no_questionnaire_item_without_one(): + assert parse_sse_lines(_OK_SSE.decode().splitlines()).clarifying_questions_item_id is None + + +def test_send_clarifying_answers_sends_the_chat_panel_shape(): + captured = [] + answers = [ + { + "questionIndex": 0, + "questionText": "Which one?", + "answer": {"type": "singleChoice", "optionIndex": 1, "optionText": "{metric/net_sales}"}, + } + ] + _capture_body_client(captured, reasoning_effort="LOW").send_clarifying_answers("conv", "i-card", answers) + assert captured == [ + { + "item": { + "role": "user", + "content": {"type": "clarifyingAnswers", "questionsItemId": "i-card", "answers": answers}, + }, + "options": {"reasoningEffort": "LOW"}, + } + ]