From 3fd94eefc7bb0d41ef4cd89b4c847da74c3bb643 Mon Sep 17 00:00:00 2001 From: huyenthanh09 Date: Tue, 3 Nov 2026 11:18:32 +0700 Subject: [PATCH 1/3] feat(gooddata-eval): grade a conversation turn's clarifying question The conversation evaluator scored only what a turn finally produced, so it could not tell an agent that asked the clarifying question the request needs from one that guessed, nor one that asked about something the request already settled. gen-ai now asks closed choices as a clarifyingQuestions part, and none of that was visible to a fixture. A turn gains four opt-in keys, read from its first complete reply before any simulated-user answer: - expects_clarification: true means the reply must ask, false means it must not. A confirmation is not a question, but a question followed by a confirmation phrase ("A or B? Please confirm.") is. - expected_clarification_form: questionnaire (a clarifyingQuestions part) or text (a question in prose only). - max_clarifying_questions: the questionnaire asks at most this many. - clarifying_options_grounded: every {type/id} option names an object of the workspace. The workspace's metrics, attributes, labels, facts, datasets, parameters, dashboards and visualizations are listed on each turn that checks, so an object an earlier turn created counts. An attribute and its label accept each other, {date/...} names a date dataset and {date_attribute/...} an attribute. gen-ai types no listing holds (computed_attribute, filter_context, document, report) stay unchecked. A mismatch fails context_success and is listed in failure_reasons, and the turn reports turn__clarification_correct to Langfuse. The last three keys need expects_clarification: true and are rejected otherwise, since they would pass without a question; the last two also fail a question asked in prose. Legacy mode never lets these checks fail an item, so a fixture with expects_clarification raises there. Without the keys nothing changes: no check, no score, the same turn detail. The entity API links a next page from every page, an empty one too, so the listing stops on a short page instead. A kind the server lacks (404) is empty. A listing that fails leaves the turn ungraded rather than passed or the run ended: no clarification_correct score, and the reason as clarification_ungraded in the turn detail. A mismatch the listing is not needed for still fails the turn. Neither the listing nor a reply the server cut short counts toward what the turn is graded on or its run latency. Checked live on an ecommerce_demo workspace with gpt-5.6-luna: three QA-30039 items (visualization, alert, anomaly detection) with all four keys set pass, and the workspace listing returns. jira: QA-30039 risk: low Co-Authored-By: Claude Opus 5.5 --- .../core/agentic/_clarification.py | 139 +++++++ .../core/agentic/_conversation_context.py | 16 + .../core/agentic/conversation.py | 120 +++++- .../tests/test_agentic_conversation.py | 365 ++++++++++++++++++ 4 files changed, 638 insertions(+), 2 deletions(-) create mode 100644 packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py new file mode 100644 index 000000000..83e12b7b1 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py @@ -0,0 +1,139 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Whether a conversation turn's first reply asked the clarifying question its fixture expects.""" + +from __future__ import annotations + +import re +from typing import Any, Literal + +import httpx +from pydantic import BaseModel, Field + +from gooddata_eval.core.models import ChatResult + +ClarificationForm = Literal["questionnaire", "text"] + +_REF_RE = re.compile(r"\{([A-Za-z_]+)/([^{}]+)\}") +# The entity kinds a `{type/id}` option reference can name. An attribute and its label are +# spelled either way by the agent, so each accepts the other. A date dataset is a dataset. +_REF_ENTITIES: dict[str, tuple[str, ...]] = { + "metric": ("metrics",), + "fact": ("facts",), + "dataset": ("datasets",), + "date": ("datasets",), + "attribute": ("attributes", "labels"), + "date_attribute": ("attributes", "labels"), + "label": ("labels", "attributes"), + "parameter": ("parameters",), + "dashboard": ("analyticalDashboards",), + "analyticalDashboard": ("analyticalDashboards",), + "visualization": ("visualizationObjects",), + "visualizationObject": ("visualizationObjects",), + "insight": ("visualizationObjects",), +} +# Reference types gen-ai writes that no entity listing holds, so grounding cannot check them. +_UNLISTED_REF_TYPES = frozenset({"computed_attribute", "filter_context", "document", "report"}) +_ENTITY_KINDS = tuple(sorted({kind for kinds in _REF_ENTITIES.values() for kind in kinds})) +_PAGE_SIZE = 1000 + + +class FirstReply(BaseModel): + """What the turn's first assistant reply asked, read before any simulated-user answer.""" + + asked: bool + form: ClarificationForm | None = None + question_count: int = 0 + option_refs: list[str] = Field(default_factory=list) + + +def read_first_reply(chat_result: ChatResult, *, asked_in_text: bool) -> FirstReply: + """The reply's clarifying questions; a ``clarifyingQuestions`` part makes it a questionnaire. + + ``asked_in_text`` is the caller's reading of the prose, for a question without a part. + """ + parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"] + questions = [q for p in parts for q in p.get("questions") or []] + options = [str(o.get("text") or "") for q in questions for o in (q.get("control") or {}).get("options") or []] + refs = [f"{kind}/{obj_id}" for text in options for kind, obj_id in _REF_RE.findall(text)] + if parts: + return FirstReply(asked=True, form="questionnaire", question_count=len(questions), option_refs=refs) + return FirstReply(asked=asked_in_text, form="text" if asked_in_text else None) + + +def fetch_object_refs( + host: str, token: str, workspace_id: str, *, transport: httpx.BaseTransport | None = None +) -> set[str]: + """Every ``/`` the workspace holds, inherited objects included. + + The entity API sends a ``next`` link on every page, an empty one too, so a short page ends the listing. + A kind the server does not know (404) has no objects. A body that is not an entity page raises ``ValueError``. + """ + found: set[str] = set() + headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} + with httpx.Client(timeout=60, headers=headers, transport=transport) as client: + for kind in _ENTITY_KINDS: + url = f"{host.rstrip('/')}/api/v1/entities/workspaces/{workspace_id}/{kind}" + page = 0 + while True: + response = client.get(url, params={"size": _PAGE_SIZE, "page": page}) + if response.status_code == httpx.codes.NOT_FOUND: + break + response.raise_for_status() + body: Any = response.json() + data = body.get("data") if isinstance(body, dict) else None + if not isinstance(data, list): + raise ValueError(f"{kind} page {page} is not an entity page") + found.update(f"{kind}/{obj['id']}" for obj in data if isinstance(obj, dict) and obj.get("id")) + if len(data) < _PAGE_SIZE: + break + page += 1 + return found + + +def ungrounded_refs(refs: list[str], known: set[str]) -> list[str]: + """The references that name no object of the workspace, or a kind it has none of. + + A type of ``_UNLISTED_REF_TYPES`` is left unchecked. + """ + missing: list[str] = [] + for ref in refs: + ref_type, _, obj_id = ref.partition("/") + if ref_type in _UNLISTED_REF_TYPES: + continue + kinds = _REF_ENTITIES.get(ref_type, ()) + if not any(f"{kind}/{obj_id}" in known for kind in kinds): + missing.append(ref) + return missing + + +def clarification_mismatches( + reply: FirstReply, + *, + expects_clarification: bool, + expected_form: ClarificationForm | None, + max_questions: int | None, + known_refs: set[str] | None, + reads_questionnaire: bool = False, +) -> list[str]: + """How the first reply differs from the expected clarification; empty when it matches. + + ``known_refs`` is None when the fixture does not ask for grounded options. + Options without a ``{type/id}`` reference are not checked for grounding. + ``reads_questionnaire`` fails a prose question, which has no options for these checks to read. + """ + if not expects_clarification: + return [f"asked a clarifying question ({reply.form}) the request does not need"] if reply.asked else [] + if not reply.asked: + return ["expected a clarifying question, but the first reply asked none"] + out: list[str] = [] + if expected_form is not None and reply.form != expected_form: + out.append(f"clarification form: expected {expected_form}, got {reply.form}") + elif reads_questionnaire and reply.form != "questionnaire": + out.append(f"the turn checks a questionnaire, but the first reply asked as {reply.form}") + if max_questions is not None and reply.question_count > max_questions: + out.append(f"asked {reply.question_count} questions, at most {max_questions} expected") + if known_refs is not None: + missing = ungrounded_refs(reply.option_refs, known_refs) + if missing: + out.append(f"options name objects the workspace does not have: {missing}") + return out diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py index 55bbd8f89..f14d4c18f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_conversation_context.py @@ -68,6 +68,8 @@ # word, so the apostrophes in "don't ... it's" do not mask the text between them. _FORMULA_SPAN_RE = re.compile(r"`+[^`]*`+|\{[^{}]*\}|\"[^\"\n]*\"|(? ReplyKind: return "no_action" +def asks_in_text(chat_result: ChatResult, rendered_text: str) -> bool: + """Whether a reply without the expected output asks the user something, read for the clarification checks. + + ``classify_reply`` reads "Did you mean A or B? Please confirm." as a confirmation, which is right for + what the simulated user says back. Here it is a question: a sentence that ends in a question mark and + is not itself a confirmation phrase asks, whatever confirmation follows it. + """ + kind = classify_reply(chat_result, rendered_text) + if kind != "confirmation" or chat_result.alert_proposals: + return kind == "question" + sentences = _SENTENCE_END_RE.split(rendered_text.strip()) + return any(s.rstrip().endswith("?") and not _CONFIRMATION_RE.search(s) for s in sentences) + + @dataclass class TranscriptEntry: role: Literal["user", "assistant"] diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 3788598a5..83b764475 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -27,11 +27,19 @@ from gooddata_sdk import GoodDataSdk from pydantic import BaseModel, Field, ValidationError, model_validator +from gooddata_eval.core.agentic._clarification import ( + ClarificationForm, + FirstReply, + clarification_mismatches, + fetch_object_refs, + read_first_reply, +) from gooddata_eval.core.agentic._conversation_context import ( CONFIRMATION_REPLY, NUDGE_MESSAGE, ClarificationJudge, TranscriptEntry, + asks_in_text, classify_reply, clip, judge_prompt, @@ -108,6 +116,12 @@ class TurnDefinition(BaseModel): and setting it to null clears it. A turn that leaves it out keeps the current one. Its clarification replies carry it too, since gen-ai treats a message without a context as cleared. + + ``expects_clarification`` grades the turn's first reply: true means it must ask, false means + it must not. Left out, nothing is checked. With true, ``expected_clarification_form`` names how + it asks (``questionnaire`` = a ``clarifyingQuestions`` part, ``text`` = prose only), + ``max_clarifying_questions`` caps the questionnaire, and ``clarifying_options_grounded`` + requires every ``{type/id}`` option to name an object of the workspace. """ turn_id: str @@ -121,6 +135,33 @@ class TurnDefinition(BaseModel): depends_on: list[str] = Field(default_factory=list) set_answers: list[str] = Field(default_factory=list) user_context: dict | None = None + expects_clarification: bool | None = None + expected_clarification_form: ClarificationForm | None = None + max_clarifying_questions: int | None = Field(default=None, ge=1) + clarifying_options_grounded: bool = False + + @property + def reads_questionnaire(self) -> bool: + """Whether a clarification key of the turn reads a ``clarifyingQuestions`` part, so prose cannot satisfy it.""" + return self.max_clarifying_questions is not None or self.clarifying_options_grounded + + @model_validator(mode="after") + def _clarification_keys_need_an_expected_question(self) -> TurnDefinition: + # Each of these only reads a question, so without one it would pass vacuously. + stated = [ + name + for name, value in ( + ("expected_clarification_form", self.expected_clarification_form), + ("max_clarifying_questions", self.max_clarifying_questions), + ("clarifying_options_grounded", self.clarifying_options_grounded or None), + ) + if value is not None + ] + if stated and self.expects_clarification is not True: + raise ValueError(f"{stated} need expects_clarification: true") + if self.reads_questionnaire and self.expected_clarification_form == "text": + raise ValueError("max_clarifying_questions and clarifying_options_grounded read a questionnaire, not text") + return self class ConversationFixture(BaseModel): @@ -196,6 +237,11 @@ class TurnResult(BaseModel): mismatches: list[str] = Field(default_factory=list) clarifications: list[ClarificationRecord] = Field(default_factory=list) depends_on: list[str] = Field(default_factory=list) + # None when the fixture states no expects_clarification, or the check could not run. + clarification_correct: bool | None = None + clarification_mismatches: list[str] = Field(default_factory=list) + # Why a stated clarification check could not run, so an ungraded turn does not read as a pass. + clarification_ungraded: str | None = None @property def skill_success(self) -> bool: @@ -211,7 +257,13 @@ def stalled(self) -> bool: @property def context_success(self) -> bool: - return self.skill_success and self.output_correct is not False and not self.lost_context and not self.stalled + return ( + self.skill_success + and self.output_correct is not False + and self.clarification_correct is not False + and not self.lost_context + and not self.stalled + ) def failure_reasons(self) -> list[str]: """Why ``context_success`` is False, in reading order; empty when it is True.""" @@ -224,6 +276,8 @@ def failure_reasons(self) -> list[str]: reasons.append(f"skill {self.expected_skill!r} not active") if self.output_correct is False: reasons.extend(self.mismatches or ["output does not match the expected output"]) + if self.clarification_correct is False: + reasons.extend(self.clarification_mismatches) if self.lost_context: reasons.append("asked for information the conversation already established") if self.stalled: @@ -258,6 +312,13 @@ def detail(self) -> dict: out = self.model_dump(include=self._DETAIL_FIELDS) out["context_success"] = self.context_success out["failure_reasons"] = self.failure_reasons() + # Only when the fixture states the check, so an item without one reports what it always did. + if self.clarification_correct is not None: + out["clarification_correct"] = self.clarification_correct + out["clarification_mismatches"] = list(self.clarification_mismatches) + elif self.clarification_ungraded is not None: + out["clarification_correct"] = None + out["clarification_ungraded"] = self.clarification_ungraded return out @@ -733,8 +794,17 @@ def run_agentic_conversation( ``user_context`` is the context attached when the conversation starts; a turn's own ``user_context`` replaces it from that turn on. + + A fixture with ``expects_clarification`` needs ``context`` mode: legacy mode does not let the + clarification checks fail the item, so it raises ``ValueError``. """ resolved_mode = resolve_conversation_mode(mode) + graded = [t.turn_id for t in fixture.turns if t.expects_clarification is not None] + if resolved_mode == "legacy" and graded: + raise ValueError( + f"turns {graded} set expects_clarification, which only context mode grades; " + f"pass mode='context' or set {CONVERSATION_MODE_ENV_VAR}=context" + ) if fresh_conversation_per_turn and initial_conversation_id is not None: raise ValueError("fresh_conversation_per_turn needs conversations this run creates itself") judge = clarification_judge if clarification_judge is not None else ClarificationJudge() @@ -786,6 +856,7 @@ def run_agentic_conversation( run_started = time.monotonic() conversation_swap_s = 0.0 # fresh_conversation_per_turn's delete/create, which is not the run's work + workspace_read_s = 0.0 # the option checks' workspace listing, which is not the run's work either for turn_index, turn in enumerate(fixture.turns): if fresh_conversation_per_turn and turn_index > 0: swap_started = time.monotonic() @@ -842,6 +913,7 @@ def run_agentic_conversation( final_result: ChatResult | None = None records: list[ClarificationRecord] = [] answers_left = list(turn.set_answers) + first_reply: FirstReply | None = None # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop # that simply runs out of range() is labelled correctly with no trailing else. @@ -910,6 +982,12 @@ def run_agentic_conversation( "assistant", " ".join(p for p in (response_text, summarize_visualizations(chat_result)) if p) ) ) + # A reply the server cut short is a stall the loop nudges past, so the next one is the first reply. + if first_reply is None and not incomplete: + asked_in_text = not _check_output_present(resolved_turn, chat_result) and asks_in_text( + chat_result, response_text + ) + first_reply = read_first_reply(chat_result, asked_in_text=asked_in_text) if _check_output_present(resolved_turn, chat_result): if incomplete: @@ -988,6 +1066,34 @@ def run_agentic_conversation( if final_result and output_present: output_correct, mismatches = _check_output_correct(resolved_turn, final_result, all_tool_calls) + # A turn with no complete reply has nothing to read; its chat error or missing output fails it. + clarification_correct: bool | None = None + clarification_problems: list[str] = [] + clarification_ungraded: str | None = None + if turn.expects_clarification is not None and first_reply is not None: + known_refs: set[str] | None = None + # Read on each such turn: an option can name an object an earlier turn created. + if turn.clarifying_options_grounded: + read_started = time.monotonic() + try: + known_refs = fetch_object_refs(host, token, workspace_id) or None + except Exception as exc: # any failure leaves the options unchecked, never passed + clarification_ungraded = f"workspace objects not readable, options left unchecked: {exc}" + print(f"[CLARIFICATION] {clarification_ungraded}") + workspace_read_s += time.monotonic() - read_started + clarification_problems = clarification_mismatches( + first_reply, + expects_clarification=turn.expects_clarification, + expected_form=turn.expected_clarification_form, + max_questions=turn.max_clarifying_questions, + known_refs=known_refs, + reads_questionnaire=turn.reads_questionnaire, + ) + if clarification_problems: + clarification_correct = False + elif clarification_ungraded is None: + clarification_correct = True + # Capture metric output for $ref resolution in subsequent turns. if final_result and turn.expected_output_type == "metric": metric_data = _extract_metric_result(all_tool_calls) @@ -1012,11 +1118,14 @@ def run_agentic_conversation( mismatches=mismatches, clarifications=records, depends_on=list(turn.depends_on), + clarification_correct=clarification_correct, + clarification_mismatches=clarification_problems, + clarification_ungraded=clarification_ungraded, ) ) # Taken here, not after the finally: its cleanup is not part of the run. - run_latency_s = time.monotonic() - run_started - conversation_swap_s + run_latency_s = time.monotonic() - run_started - conversation_swap_s - workspace_read_s finally: if owns_conversation and conversation_id: client.delete_conversation(conversation_id) @@ -1208,6 +1317,13 @@ def _write_scores(ctx: RunTraceContext) -> None: value=float(tr.skill_success), data_type="BOOLEAN", ) + if tr.clarification_correct is not None: + ctx.score( + tid, + name=f"turn_{tr.turn_id}_clarification_correct", + value=float(tr.clarification_correct), + data_type="BOOLEAN", + ) strict_checks = { "conversation_success": result.conversation_success, "full_skill_coverage": result.full_skill_coverage, diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index 37907e4cc..cef232a23 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -6,8 +6,10 @@ from datetime import date from unittest.mock import MagicMock, patch +import httpx import pytest from gooddata_eval.core.agentic import _conversation_context as ctx_mod +from gooddata_eval.core.agentic._clarification import fetch_object_refs, read_first_reply, ungrounded_refs from gooddata_eval.core.agentic._conversation_context import ( CONFIRMATION_REPLY, NO_ANSWER_REPLY, @@ -15,6 +17,7 @@ ClarificationJudge, ClarificationVerdict, _ClarificationLLM, + asks_in_text, classify_reply, judge_prompt, reply_from_facts, @@ -2269,3 +2272,365 @@ def test_a_fixture_level_user_context_is_rejected() -> None: "turns": [{"turn_id": "t1", "message": "m", "expected_skill": "visualization"}], } ) + + +# Clarification expectations: the first reply of a turn, graded against expects_clarification. +# The questionnaire shape is trimmed from a gen-ai reply on the ecommerce_demo eval workspace. + + +def _questionnaire(*questions: list[str]) -> dict: + return { + "type": "clarifyingQuestions", + "questions": [ + { + "text": f"Question {i}?", + "control": {"type": "singleChoice", "options": [{"text": o} for o in options]}, + } + for i, options in enumerate(questions) + ], + } + + +_RETURNS_CARD = _questionnaire(["{metric/returns}", "{metric/total_returns}"]) +_RETURNS_VIZ = _viz(metric="metric/total_returns", dim="label/product_category") + + +def _card_reply(*questionnaires: dict) -> ChatResult: + return _result(text="I found two return measures.", tools=[_SKILLS], unhandledParts=list(questionnaires)) + + +def _clarifying_turn(**kw: object) -> TurnDefinition: + return _viz_turn( + "t1", + expected=_RETURNS_VIZ, + message="Show me returns by product category.", + set_answers=["Use Total Returns."], + **kw, + ) + + +def test_a_turn_without_expects_clarification_is_not_graded_on_its_question() -> None: + result, _ = _run([_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [_clarifying_turn()]) + turn = result.turn_results[0] + assert turn.clarification_correct is None + assert turn.context_success is True + + +def test_an_expected_questionnaire_that_is_asked_passes() -> None: + turn_def = _clarifying_turn( + expects_clarification=True, expected_clarification_form="questionnaire", max_clarifying_questions=4 + ) + result, client = _run([_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [turn_def]) + turn = result.turn_results[0] + assert turn.clarification_correct is True + assert turn.clarification_mismatches == [] + assert turn.context_success is True + assert client.send_message.call_args_list[1].args[1] == "Use Total Returns." + assert turn.detail()["clarification_correct"] is True + + +def test_an_expected_question_that_is_not_asked_fails_the_turn() -> None: + result, _ = _run([_result(viz=_RETURNS_VIZ, tools=[_SKILLS])], [_clarifying_turn(expects_clarification=True)]) + turn = result.turn_results[0] + assert turn.output_correct is True + assert turn.clarification_correct is False + assert turn.context_success is False + assert "expected a clarifying question, but the first reply asked none" in turn.failure_reasons() + + +def test_a_question_the_request_does_not_need_fails_the_turn() -> None: + result, _ = _run( + [_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [_clarifying_turn(expects_clarification=False)] + ) + turn = result.turn_results[0] + assert turn.clarification_correct is False + assert turn.clarification_mismatches == ["asked a clarifying question (questionnaire) the request does not need"] + + +def test_a_confirmation_is_not_a_question_the_request_does_not_need() -> None: + result, _ = _run( + [_result(text="I will chart Total Returns. Please confirm.", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], + [_clarifying_turn(expects_clarification=False)], + ) + assert result.turn_results[0].clarification_correct is True + + +def test_a_prose_question_fails_an_expected_questionnaire() -> None: + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="questionnaire") + result, _ = _run( + [_result(text="Which returns measure do you want?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def] + ) + turn = result.turn_results[0] + assert turn.clarification_correct is False + assert turn.clarification_mismatches == ["clarification form: expected questionnaire, got text"] + + +def test_a_prose_question_passes_an_expected_text_question() -> None: + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="text") + result, _ = _run( + [_result(text="Which time range do you want?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def] + ) + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + ("text", "asks"), + [ + ("Did you mean Net Sales or Gross Sales? Please confirm.", True), + ("Which time range do you want?", True), + ("I will chart Total Returns. Please confirm.", False), + ("Should I proceed with Net Sales?", False), + ("Here is the plan. Shall I create it?", False), + ], +) +def test_a_question_followed_by_a_confirmation_phrase_still_asks(text: str, asks: bool) -> None: + assert asks_in_text(_result(text=text), text) is asks + + +def test_a_prose_choice_with_please_confirm_passes_an_expected_text_question() -> None: + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="text") + text = "Did you mean Returns or Total Returns? Please confirm." + result, _ = _run([_result(text=text, tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True + + +def test_more_questions_than_allowed_fail_the_turn() -> None: + card = _questionnaire(["{metric/returns}", "{metric/total_returns}"], ["Monthly", "Quarterly"]) + turn_def = _clarifying_turn(expects_clarification=True, max_clarifying_questions=1) + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_mismatches == ["asked 2 questions, at most 1 expected"] + + +def test_an_option_naming_an_object_the_workspace_lacks_fails_grounding() -> None: + card = _questionnaire(["{metric/total_returns}", "{dashboard/2. Sales}", "Plain text option"]) + known = {"metrics/total_returns", "analyticalDashboards/bf439696"} + turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) + with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", return_value=known) as fetch: + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + fetch.assert_called_once_with("h", "t", "ws") + assert result.turn_results[0].clarification_mismatches == [ + "options name objects the workspace does not have: ['dashboard/2. Sales']" + ] + + +def test_grounded_options_pass_and_the_catalog_is_read_on_each_turn_that_checks_options() -> None: + turns = [ + _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True), + _viz_turn( + "t2", + expected=_RETURNS_VIZ, + message="Again for returns.", + set_answers=["Use Total Returns."], + expects_clarification=True, + clarifying_options_grounded=True, + ), + ] + known = {"metrics/returns", "metrics/total_returns"} + with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", return_value=known) as fetch: + result, _ = _run( + [ + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ), + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ), + ], + turns, + ) + assert fetch.call_count == 2 + assert [t.clarification_correct for t in result.turn_results] == [True, True] + + +@pytest.mark.parametrize("error", [httpx.ConnectError("refused"), ValueError("metrics page 0 is not an entity page")]) +def test_an_unreadable_listing_leaves_the_turn_ungraded_rather_than_passed(error: Exception) -> None: + turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) + with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", side_effect=error): + result, _ = _run([_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [turn_def]) + turn = result.turn_results[0] + assert turn.clarification_correct is None + assert turn.clarification_ungraded is not None + assert turn.clarification_ungraded.startswith("workspace objects not readable, options left unchecked") + detail = turn.detail() + assert detail["clarification_correct"] is None + assert detail["clarification_ungraded"] == turn.clarification_ungraded + + +def test_an_unreadable_listing_still_fails_a_check_that_does_not_need_it() -> None: + turn_def = _clarifying_turn( + expects_clarification=True, clarifying_options_grounded=True, max_clarifying_questions=1 + ) + card = _questionnaire(["{metric/returns}", "{metric/total_returns}"], ["Monthly", "Quarterly"]) + with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", side_effect=ValueError("x")): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is False + + +@pytest.mark.parametrize( + ("ref", "known", "grounded"), + [ + ("metric/total_returns", {"metrics/total_returns"}, True), + ("attribute/product_category", {"labels/product_category"}, True), + ("label/date.month", {"attributes/date.month"}, True), + ("dashboard/bf439696", {"analyticalDashboards/bf439696"}, True), + ("metric/return_rate", {"metrics/total_returns"}, False), + ("widget/x", {"metrics/x"}, False), + # gen-ai's own reference types: a date is a date dataset, a date attribute an attribute. + ("date/order_date", {"datasets/order_date"}, True), + ("date_attribute/order_date.month", {"attributes/order_date.month"}, True), + ("parameter/discount", {"metrics/discount"}, False), + ("computed_attribute/sales_bucket", set(), True), + ], +) +def test_ungrounded_refs_accept_each_spelling_of_an_object(ref: str, known: set[str], grounded: bool) -> None: + assert (ungrounded_refs([ref], known) == []) is grounded + + +def test_read_first_reply_counts_questions_and_collects_option_refs() -> None: + card = _questionnaire(["{metric/returns}", "Returned amount {metric/total_returns}"], ["Monthly", "Weekly"]) + reply = read_first_reply(_result(unhandledParts=[card]), asked_in_text=False) + assert reply.form == "questionnaire" + assert reply.question_count == 2 + assert reply.option_refs == ["metric/returns", "metric/total_returns"] + + +def test_read_first_reply_reads_a_reference_type_with_an_underscore() -> None: + card = _questionnaire(["{date_attribute/order_date.month}", "{date_attribute/ship_date.month}"]) + reply = read_first_reply(_result(unhandledParts=[card]), asked_in_text=False) + assert reply.option_refs == ["date_attribute/order_date.month", "date_attribute/ship_date.month"] + + +def test_clarification_checks_in_legacy_mode_are_refused_before_any_request() -> None: + with pytest.raises(ValueError, match=r"turns \['t1'\] set expects_clarification, which only context mode grades"): + _run([], [_clarifying_turn(expects_clarification=True)], mode="legacy") + + +def test_option_keys_fail_a_question_asked_in_prose() -> None: + turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) + with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", return_value={"metrics/total_returns"}): + result, _ = _run( + [_result(text="Do you mean returns or total returns?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], + [turn_def], + ) + assert result.turn_results[0].clarification_mismatches == [ + "the turn checks a questionnaire, but the first reply asked as text" + ] + + +def test_a_stalled_reply_is_not_the_first_reply_the_question_is_read_from() -> None: + stall = TurnIncompleteError("SSE error 502", status_code=502, reason="max_iterations") + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="questionnaire") + result, _ = _run([stall, _card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + "keys", + [ + {"expected_clarification_form": "questionnaire"}, + {"max_clarifying_questions": 2}, + {"clarifying_options_grounded": True}, + {"expects_clarification": False, "max_clarifying_questions": 2}, + ], +) +def test_a_clarification_key_without_an_expected_question_is_rejected(keys: dict) -> None: + with pytest.raises(ValueError, match="need expects_clarification: true"): + TurnDefinition(turn_id="t1", message="m", expected_skill="visualization", **keys) + + +def test_a_questionnaire_key_on_a_text_question_is_rejected() -> None: + with pytest.raises(ValueError, match="read a questionnaire, not text"): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + expected_clarification_form="text", + max_clarifying_questions=2, + ) + + +def test_a_zero_question_cap_is_rejected() -> None: + with pytest.raises(ValueError): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + max_clarifying_questions=0, + ) + + +def test_the_clarification_verdict_is_written_to_langfuse_only_when_stated() -> None: + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)] + captured: dict = {} + + def _capture(_submit: object, _identity: object, **kwargs: object) -> None: + captured["write_scores"] = kwargs["write_scores"] + + fixture = ConversationFixture( + id="c", expected_skills=["visualization"], turns=[_clarifying_turn(expects_clarification=True)] + ) + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + patch("gooddata_eval.core.agentic.conversation.submit_trace_scoring", _capture), + ): + evaluate_agentic_conversation( + host="h", + token="t", + workspace_id="ws", + fixture=fixture, + langfuse=MagicMock(), + dataset_item_id="item-1", + mode="context", + ) + ctx = MagicMock() + captured["write_scores"](ctx) + scores = {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + assert scores["turn_t1_clarification_correct"] == 1.0 + + +def test_fetch_object_refs_ends_on_a_short_page_although_every_page_links_a_next_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("gooddata_eval.core.agentic._clarification._PAGE_SIZE", 2) + pages = {("metrics", 0): ["a", "b"], ("metrics", 1): ["c"]} + seen: list[tuple[str, int]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + kind = request.url.path.rsplit("/", 1)[1] + page = int(request.url.params["page"]) + seen.append((kind, page)) + if kind == "parameters": + return httpx.Response(404, json={"title": "Not Found"}) + assert request.headers["Authorization"] == "Bearer t" + ids = pages.get((kind, page), []) + nxt = f"{request.url.copy_with(params={'page': page + 1, 'size': 2})}" + data: list = [{"id": i} for i in ids] + if (kind, page) == ("metrics", 0): + data += ["junk", {"attributes": {"title": "No id"}}] + return httpx.Response(200, json={"data": data, "links": {"next": nxt}}) + + refs = fetch_object_refs("https://h/", "t", "ws", transport=httpx.MockTransport(handler)) + assert {r for r in refs if r.startswith("metrics/")} == {"metrics/a", "metrics/b", "metrics/c"} + assert [p for k, p in seen if k == "metrics"] == [0, 1] + assert max(p for _, p in seen) == 1 + assert not any(r.startswith("parameters/") for r in refs) + + +@pytest.mark.parametrize( + "response", + [ + httpx.Response(200, text="Sign in"), + httpx.Response(200, json=[{"id": "a"}]), + httpx.Response(200, json={"errors": ["boom"]}), + ], + ids=["not-json", "a-list", "no-data"], +) +def test_fetch_object_refs_refuses_a_body_that_is_not_an_entity_page(response: httpx.Response) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return response + + with pytest.raises(ValueError): + fetch_object_refs("https://h/", "t", "ws", transport=httpx.MockTransport(handler)) From e4a5bbc1ebf2e33d29bf5aaf33e51128bac09a38 Mon Sep 17 00:00:00 2001 From: huyenthanh09 Date: Tue, 3 Nov 2026 16:50:00 +0700 Subject: [PATCH 2/3] feat(gooddata-eval): answer a questionnaire with clarifyingAnswers The conversation evaluator answered every clarifying question with text, so it never sent what the chat panel sends: a clarifyingAnswers message with the picked positions and the texts shown, where a question left out is skipped. Skipping, and what the agent does after a skip, could not be evaluated. ChatResult keeps the itemId of the assistant item holding a clarifyingQuestions part, and each such part keeps its item's id, since gen-ai numbers questions within the item an answer names. ChatClient.send_clarifying_answers sends the message with it as questionsItemId. send_message and the new method share one request path, so options and user context apply to both. A turn gains opt-in keys, all needing expects_clarification: true: - clarifying_answers: picks for the questionnaire of the turn's first reply, sent even when the judge flags that question as lost context, since the fixture expects it. A pick names an option by a type/id reference or its whole text, or lists both spellings, since the agent writes options either way; it may name the question's position. attribute/x and label/x match each other. A question no pick matches is skipped, and an empty list skips them all. Later questions, a follow-up confirmation included, still get set_answers text. A pick no questionnaire offered fails the turn, and so do answers never sent, with the reason: the first reply asked no questionnaire, delivered the output with it, had no itemId, or the clarification budget ended the turn. - states_skipped_assumption: a reply after the answers names an option of each skipped question, by a whole reference or its text as whole words; the reply asking for confirmation usually states it. A mention stands in for the stated assumption. - skipped_questions_not_reasked: no later questionnaire asks a skipped question again, judged by its text or its option set. Yes/No style options fit any question, a confirmation too, so they count only with the same text. - expects_confirmation_after_answers: the reply to the answers is not the output. A skip is not an approval, so the agent stops first. The transcript records the answers as the text gen-ai builds for the model from them. Without the keys nothing changes. Checked live on an ecommerce_demo workspace with gpt-5.6-luna: a picked sales measure is sent and charted, skipping every question leads to a stated assumption with no re-ask, and a dashboard build stops for confirmation after a skip. jira: QA-30039 risk: low Co-Authored-By: Claude Opus 5.5 --- .../core/agentic/_clarification.py | 195 ++++++++- .../core/agentic/conversation.py | 114 ++++- .../src/gooddata_eval/core/chat/sse_client.py | 34 +- .../src/gooddata_eval/core/models.py | 3 + .../tests/test_agentic_conversation.py | 388 +++++++++++++++++- .../gooddata-eval/tests/test_sse_client.py | 46 +++ 6 files changed, 768 insertions(+), 12 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py index 83e12b7b1..836d3a11e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py @@ -7,7 +7,7 @@ from typing import Any, Literal import httpx -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from gooddata_eval.core.models import ChatResult @@ -33,8 +33,14 @@ } # Reference types gen-ai writes that no entity listing holds, so grounding cannot check them. _UNLISTED_REF_TYPES = frozenset({"computed_attribute", "filter_context", "document", "report"}) +_GENERIC_ANSWERS = frozenset( + {"yes", "no", "ok", "okay", "cancel", "confirm", "continue", "proceed", "skip", "not now", "go ahead"} +) _ENTITY_KINDS = tuple(sorted({kind for kinds in _REF_ENTITIES.values() for kind in kinds})) _PAGE_SIZE = 1000 +# A shorter skipped option, such as "No" or "Q1", also occurs in text that assumes nothing, so it +# never counts as named. +_MIN_NAMED_OPTION_CHARS = 3 class FirstReply(BaseModel): @@ -137,3 +143,190 @@ def clarification_mismatches( if missing: out.append(f"options name objects the workspace does not have: {missing}") return out + + +class ClarifyingPick(BaseModel): + """An option the fixture picks in a questionnaire. + + ``option`` is a ``type/id`` reference the option names, or the option's whole text. A list names + the spellings of one option, since the agent writes it as a reference or as plain text. + ``question`` limits the pick to that position; without it, the first question offering the + option takes it. + """ + + option: str | list[str] + question: int | None = Field(default=None, ge=0) + + @property + def spellings(self) -> list[str]: + return [self.option] if isinstance(self.option, str) else list(self.option) + + @property + def label(self) -> str: + return " | ".join(self.spellings) + + @model_validator(mode="after") + def _names_an_option(self) -> ClarifyingPick: + if not self.spellings or not all(s.strip() for s in self.spellings): + raise ValueError("option needs at least one non-empty spelling") + return self + + +class ClarifyingAnswers(BaseModel): + """The ``clarifyingAnswers`` for one questionnaire, and what they left out.""" + + answers: list[dict] = Field(default_factory=list) + skipped: list[dict] = Field(default_factory=list) + matched_picks: list[str] = Field(default_factory=list) + + def render(self) -> str: + """The text gen-ai builds for the model from these answers: every question, a skipped one as Skipped.""" + sections = [(a["questionIndex"], a["questionText"], a["answer"]["optionText"]) for a in self.answers] + sections += [(s["index"], s["text"], "Skipped") for s in self.skipped] + return "\n\n".join(f"**{text}**\n\n{answer}" for _, text, answer in sorted(sections, key=lambda s: s[0])) + + +def _entity_refs(ref: str) -> set[str]: + ref_type, _, obj_id = ref.partition("/") + return {f"{kind}/{obj_id}" for kind in _REF_ENTITIES.get(ref_type, ())} + + +def _all_questions(chat_result: ChatResult) -> list[dict]: + parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"] + return [q for p in parts for q in p.get("questions") or []] + + +def _questions(chat_result: ChatResult) -> list[dict]: + """The questions of the item the answers name, numbered as gen-ai numbers them: across that item's parts.""" + item_id = chat_result.clarifying_questions_item_id + parts = [ + p + for p in chat_result.unhandled_parts or [] + if p.get("type") == "clarifyingQuestions" and p.get("itemId") in (None, item_id) + ] + return [q for p in parts for q in p.get("questions") or []] + + +def _options(question: dict) -> list[str]: + return [str(o.get("text") or "") for o in (question.get("control") or {}).get("options") or []] + + +def _plain(text: str) -> str: + return " ".join(_REF_RE.sub(" ", text).lower().split()) + + +def _same_object(ref: str, other: str) -> bool: + return ref == other or bool(_entity_refs(ref) & _entity_refs(other)) + + +def option_matches(option_text: str, wanted: str) -> bool: + """Whether an option is the one a pick names, by its ``{type/id}`` reference or its whole text. + + A reference matches the spellings grounding accepts for the same object, such as ``attribute/x`` and ``label/x``. + """ + ref = wanted.strip().strip("{}") + if "/" in ref and any(_same_object(ref, f"{kind}/{obj_id}") for kind, obj_id in _REF_RE.findall(option_text)): + return True + return option_text.strip().lower() == wanted.strip().lower() + + +def build_clarifying_answers(chat_result: ChatResult, picks: list[ClarifyingPick]) -> ClarifyingAnswers: + """Answer the reply's questionnaire with the picks; a question no pick matches is skipped.""" + questions = _questions(chat_result) + out = ClarifyingAnswers() + answered: set[int] = set() + for pick in picks: + for qi, question in enumerate(questions): + if qi in answered or (pick.question is not None and pick.question != qi): + continue + oi = next( + ( + i + for i, text in enumerate(_options(question)) + if any(option_matches(text, spelling) for spelling in pick.spellings) + ), + None, + ) + if oi is None: + continue + answered.add(qi) + out.matched_picks.append(pick.label) + out.answers.append( + { + "questionIndex": qi, + "questionText": question.get("text") or "", + "answer": {"type": "singleChoice", "optionIndex": oi, "optionText": _options(question)[oi]}, + } + ) + break + out.answers.sort(key=lambda a: a["questionIndex"]) + out.skipped = [ + {"index": qi, "text": q.get("text") or "", "options": _options(q)} + for qi, q in enumerate(questions) + if qi not in answered + ] + return out + + +def _names_option(text: str, option: str) -> bool: + """Whether ``text`` names the option: one of its references whole, or its words as whole words.""" + refs_in_text = [f"{kind}/{obj_id}" for kind, obj_id in _REF_RE.findall(text)] + for kind, obj_id in _REF_RE.findall(option): + if any(_same_object(f"{kind}/{obj_id}", ref) for ref in refs_in_text): + return True + label = _plain(option) + return ( + len(label) >= _MIN_NAMED_OPTION_CHARS + and re.search(rf"(? bool: + """Whether ``question`` asks ``skipped`` again: the same text, or the same options that name something. + + Options such as Yes and No fit any question, a confirmation too, so they count only with the same text. + """ + if _plain(question.get("text") or "") == _plain(skipped["text"]): + return True + options = sorted(_options(question)) + specific = any(_plain(option).strip(".!") not in _GENERIC_ANSWERS for option in options) + return specific and options == sorted(skipped["options"]) + + +def after_answers_mismatches( + *, + unmatched_picks: list[str], + skipped: list[dict], + replies_after: list[tuple[str, ChatResult, bool]], + states_skipped_assumption: bool, + skipped_questions_not_reasked: bool, + expects_confirmation_after_answers: bool, +) -> list[str]: + """How the turn after its structured answers differs from the fixture; empty when it matches. + + ``replies_after`` holds each later reply with its kind (``output`` or a ``classify_reply`` kind) and + whether it answers a ``clarifyingAnswers`` message directly. + + ``states_skipped_assumption`` stands in for "the agent says what it assumed": it passes when any reply + after the answers names an option of each skipped question. It cannot tell an assumption from a + mention, and it does not need the assumption in the final reply, since a reply that asks for + confirmation usually states it. + """ + out = [f"no questionnaire offered the picked option {pick!r}" for pick in unmatched_picks] + if states_skipped_assumption: + texts = [reply.text_response or "" for _, reply, _ in replies_after] + out.extend( + f"no reply after the answers names an option of the skipped question {question['text']!r}" + for question in skipped + if not any(_names_option(text, option) for text in texts for option in question["options"]) + ) + if skipped_questions_not_reasked: + for question in skipped: + asked_again = any( + _same_question(later, question) for _, reply, _ in replies_after for later in _all_questions(reply) + ) + if asked_again: + out.append(f"asked the skipped question {question['text']!r} again") + if expects_confirmation_after_answers and any(kind == "output" and follows for kind, _, follows in replies_after): + out.append("acted on the answers in the same reply, without waiting for confirmation") + return out diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 83b764475..01eeb6c2d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -29,7 +29,10 @@ from gooddata_eval.core.agentic._clarification import ( ClarificationForm, + ClarifyingPick, FirstReply, + after_answers_mismatches, + build_clarifying_answers, clarification_mismatches, fetch_object_refs, read_first_reply, @@ -42,6 +45,7 @@ asks_in_text, classify_reply, clip, + has_structured_question, judge_prompt, render_transcript, reply_from_facts, @@ -122,6 +126,16 @@ class TurnDefinition(BaseModel): it asks (``questionnaire`` = a ``clarifyingQuestions`` part, ``text`` = prose only), ``max_clarifying_questions`` caps the questionnaire, and ``clarifying_options_grounded`` requires every ``{type/id}`` option to name an object of the workspace. + + ``clarifying_answers`` answers the questionnaire of the turn's first reply the way the chat panel + does, with a ``clarifyingAnswers`` message: a question no pick matches is skipped, and an empty + list skips them all. Every other question, a later questionnaire included, gets ``set_answers`` + text, so a follow-up confirmation can be answered. With it, + ``states_skipped_assumption`` requires a reply after the answers to name an option of each skipped + question (a mention, which stands in for the stated assumption), + ``skipped_questions_not_reasked`` forbids a later questionnaire asking one again, and + ``expects_confirmation_after_answers`` requires the agent to stop after each ``clarifyingAnswers`` + message rather than act on it in the same reply: a skip is not an approval. """ turn_id: str @@ -139,11 +153,19 @@ class TurnDefinition(BaseModel): expected_clarification_form: ClarificationForm | None = None max_clarifying_questions: int | None = Field(default=None, ge=1) clarifying_options_grounded: bool = False + clarifying_answers: list[ClarifyingPick] | None = None + states_skipped_assumption: bool = False + skipped_questions_not_reasked: bool = False + expects_confirmation_after_answers: bool = False @property def reads_questionnaire(self) -> bool: """Whether a clarification key of the turn reads a ``clarifyingQuestions`` part, so prose cannot satisfy it.""" - return self.max_clarifying_questions is not None or self.clarifying_options_grounded + return ( + self.max_clarifying_questions is not None + or self.clarifying_options_grounded + or self.clarifying_answers is not None + ) @model_validator(mode="after") def _clarification_keys_need_an_expected_question(self) -> TurnDefinition: @@ -154,13 +176,28 @@ def _clarification_keys_need_an_expected_question(self) -> TurnDefinition: ("expected_clarification_form", self.expected_clarification_form), ("max_clarifying_questions", self.max_clarifying_questions), ("clarifying_options_grounded", self.clarifying_options_grounded or None), + ("clarifying_answers", self.clarifying_answers), ) if value is not None ] if stated and self.expects_clarification is not True: raise ValueError(f"{stated} need expects_clarification: true") if self.reads_questionnaire and self.expected_clarification_form == "text": - raise ValueError("max_clarifying_questions and clarifying_options_grounded read a questionnaire, not text") + raise ValueError( + "max_clarifying_questions, clarifying_options_grounded and clarifying_answers read a " + "questionnaire, not text" + ) + after_answers = [ + name + for name, value in ( + ("states_skipped_assumption", self.states_skipped_assumption), + ("skipped_questions_not_reasked", self.skipped_questions_not_reasked), + ("expects_confirmation_after_answers", self.expects_confirmation_after_answers), + ) + if value + ] + if after_answers and self.clarifying_answers is None: + raise ValueError(f"{after_answers} read the turn after its clarifying_answers; set clarifying_answers") return self @@ -795,8 +832,8 @@ def run_agentic_conversation( ``user_context`` is the context attached when the conversation starts; a turn's own ``user_context`` replaces it from that turn on. - A fixture with ``expects_clarification`` needs ``context`` mode: legacy mode does not let the - clarification checks fail the item, so it raises ``ValueError``. + A fixture with ``expects_clarification`` needs ``context`` mode: legacy mode neither sends + ``clarifying_answers`` nor lets the clarification checks fail the item, so it raises ``ValueError``. """ resolved_mode = resolve_conversation_mode(mode) graded = [t.turn_id for t in fixture.turns if t.expects_clarification is not None] @@ -914,6 +951,14 @@ def run_agentic_conversation( records: list[ClarificationRecord] = [] answers_left = list(turn.set_answers) first_reply: FirstReply | None = None + # Structured answers: the one to send next, what they skipped, and every reply after them. + pending_answers: tuple[str, list[dict]] | None = None + answers_sent = False + # Why clarifying_answers were not sent, when the loop knows it better than what is left after. + answers_unsent: str | None = None + matched_picks: set[str] = set() + skipped_questions: list[dict] = [] + replies_after_answers: list[tuple[str, ChatResult, bool]] = [] # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop # that simply runs out of range() is labelled correctly with no trailing else. @@ -921,8 +966,19 @@ def run_agentic_conversation( for _iter in range(max_clarification_turns + 1): transcript.append(TranscriptEntry("user", current_message)) incomplete = False + follows_answers = pending_answers is not None + is_first_reply = False try: - chat_result = client.send_message(conversation_id, current_message, user_context=current_context) + if pending_answers is not None: + questions_item_id, answers = pending_answers + pending_answers = None + chat_result = client.send_clarifying_answers( + conversation_id, questions_item_id, answers, user_context=current_context + ) + else: + chat_result = client.send_message( + conversation_id, current_message, user_context=current_context + ) except TurnIncompleteError as exc: chat_result = exc.partial_result or ChatResult() incomplete = True @@ -988,10 +1044,24 @@ def run_agentic_conversation( chat_result, response_text ) first_reply = read_first_reply(chat_result, asked_in_text=asked_in_text) + is_first_reply = True + # The questionnaire clarifying_answers would answer, when this reply carries it. + answerable = ( + turn.clarifying_answers is not None and is_first_reply and has_structured_question(chat_result) + ) + if answers_sent: + reply_kind = ( + "output" + if _check_output_present(resolved_turn, chat_result) + else classify_reply(chat_result, response_text) + ) + replies_after_answers.append((reply_kind, chat_result, follows_answers)) if _check_output_present(resolved_turn, chat_result): if incomplete: records.append(ClarificationRecord(kind="turn_incomplete", agent_message=clip(response_text))) + if answerable: + answers_unsent = "the first reply delivered the output with its questionnaire, so nothing waited on the answers" turn_exit = LoopExit.SUCCESS break @@ -1026,6 +1096,11 @@ def run_agentic_conversation( and any(r.kind in ("no_action", "turn_incomplete") for r in records) ) if clarification_turns >= max_clarification_turns or second_stall: + if answerable: + answers_unsent = ( + f"the clarification budget ({max_clarification_turns}) ended the turn " + "before clarifying_answers were sent" + ) records.append(record) break @@ -1035,6 +1110,15 @@ def run_agentic_conversation( reply = CONFIRMATION_REPLY elif kind in ("no_action", "turn_incomplete"): reply = NUDGE_MESSAGE + # Before restating: the fixture expects this question, so a lost_context verdict on it + # must not send the picks to a later questionnaire. + elif answerable and turn.clarifying_answers is not None and chat_result.clarifying_questions_item_id: + built = build_clarifying_answers(chat_result, turn.clarifying_answers) + matched_picks.update(built.matched_picks) + skipped_questions = built.skipped + pending_answers = (chat_result.clarifying_questions_item_id, built.answers) + answers_sent = True + reply = built.render() elif record.lost_context is True and not any(r.lost_context is True for r in records): # Once per turn, so a judge mistake cannot loop the user on restating. reply = reply_restating(render_transcript(transcript[:-1]), response_text) @@ -1042,6 +1126,8 @@ def run_agentic_conversation( reply = answers_left.pop(0) else: reply = reply_from_facts(render_transcript(transcript[:-1]), response_text, turn.set_answers) + if answerable and not answers_sent: + answers_unsent = "the first reply's questionnaire has no itemId to send clarifying_answers to" record.reply = clip(reply) records.append(record) @@ -1089,6 +1175,24 @@ def run_agentic_conversation( known_refs=known_refs, reads_questionnaire=turn.reads_questionnaire, ) + if turn.clarifying_answers is not None and not answers_sent: + # Every check after the answers would pass with nothing to read. + if answers_unsent is None: + answers_unsent = ( + "the first reply asked no questionnaire to answer" + if first_reply.form != "questionnaire" + else "the turn ended before clarifying_answers were sent" + ) + clarification_problems.append(f"clarifying_answers not sent: {answers_unsent}") + if turn.clarifying_answers is not None and answers_sent: + clarification_problems += after_answers_mismatches( + unmatched_picks=[p.label for p in turn.clarifying_answers if p.label not in matched_picks], + skipped=skipped_questions, + replies_after=replies_after_answers, + states_skipped_assumption=turn.states_skipped_assumption, + skipped_questions_not_reasked=turn.skipped_questions_not_reasked, + expects_confirmation_after_answers=turn.expects_confirmation_after_answers, + ) if clarification_problems: clarification_correct = False elif clarification_ungraded is None: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 0753f52b2..ce7ead5c1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -206,6 +206,7 @@ class _SseAccumulator: alert_proposals: list[dict[str, Any]] = field(default_factory=list) search_results: list[dict[str, Any]] = field(default_factory=list) unhandled_parts: list[dict[str, Any]] = field(default_factory=list) + clarifying_questions_item_id: str | None = None tool_call_events: list[dict[str, Any]] = field(default_factory=list) call_id_to_event_index: dict[str, int] = field(default_factory=dict) reasoning_steps: list[dict[str, Any]] = field(default_factory=list) @@ -226,7 +227,8 @@ def _handle_text(content: dict[str, Any], acc: _SseAccumulator) -> None: acc.text_parts.append(text) -def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator) -> None: +def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator, item_id: str | None = None) -> None: + """Sort the item's parts; a ``clarifyingQuestions`` part keeps ``item_id``, since answers number questions per item.""" for part in content.get("parts", []): ptype = part.get("type") if ptype == "text": @@ -247,7 +249,8 @@ def _handle_multipart(content: dict[str, Any], acc: _SseAccumulator) -> None: else: if ptype not in _KNOWN_PART_TYPES: _log.warning("unknown multipart part type %r; captured as an unhandled part", ptype) - acc.unhandled_parts.append(part) + tagged = ptype == "clarifyingQuestions" and item_id + acc.unhandled_parts.append({**part, "itemId": item_id} if tagged else part) def _handle_reasoning(content: dict[str, Any], acc: _SseAccumulator) -> None: @@ -300,6 +303,7 @@ def _build_chat_result(acc: _SseAccumulator) -> ChatResult: "alertProposals": acc.alert_proposals, "searchResults": acc.search_results, "unhandledParts": acc.unhandled_parts, + "clarifyingQuestionsItemId": acc.clarifying_questions_item_id, "toolCallEvents": acc.tool_call_events, "reasoningStepCount": len(acc.reasoning_steps), "reasoningSteps": [step["summary"] for step in acc.reasoning_steps], @@ -439,7 +443,9 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult: if ctype == "text": _handle_text(content, acc) elif ctype == "multipart": - _handle_multipart(content, acc) + _handle_multipart(content, acc, item_id=item.get("itemId")) + if any(p.get("type") == "clarifyingQuestions" for p in content.get("parts", [])): + acc.clarifying_questions_item_id = item.get("itemId") or acc.clarifying_questions_item_id elif ctype == "reasoning": _handle_reasoning(content, acc) elif ctype == "toolCall": @@ -540,11 +546,31 @@ def delete_conversation(self, conversation_id: str) -> None: def send_message( self, conversation_id: str, question: str, *, user_context: dict[str, Any] | None = None + ) -> ChatResult: + return self._send(conversation_id, {"type": "text", "text": question}, user_context=user_context) + + def send_clarifying_answers( + self, + conversation_id: str, + questions_item_id: str, + answers: list[dict[str, Any]], + *, + user_context: dict[str, Any] | None = None, + ) -> ChatResult: + """Answer a questionnaire the way the chat panel does: positions plus the texts shown. + + A question left out of ``answers`` is skipped; an empty list skips them all. + """ + content = {"type": "clarifyingAnswers", "questionsItemId": questions_item_id, "answers": answers} + return self._send(conversation_id, content, user_context=user_context) + + def _send( + self, conversation_id: str, content: dict[str, Any], *, user_context: dict[str, Any] | None = None ) -> ChatResult: url = f"{self._base}/{conversation_id}/messages" join_headers, joined = self._join_item_trace(conversation_id) headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json", **join_headers} - body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}} + body: dict[str, Any] = {"item": {"role": "user", "content": content}} options: dict[str, str] = {} if self._reasoning_effort is not None: options["reasoningEffort"] = self._reasoning_effort diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index fb260099f..85f92e74c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -351,6 +351,9 @@ class ChatResult(BaseModel): alert_proposals: list[dict] = Field(default_factory=list, alias="alertProposals") search_results: list[dict] = Field(default_factory=list, alias="searchResults") unhandled_parts: list[dict] = Field(default_factory=list, alias="unhandledParts") + # The itemId of the assistant item holding a clarifyingQuestions part: a clarifyingAnswers + # message names it as questionsItemId. + clarifying_questions_item_id: str | None = Field(default=None, alias="clarifyingQuestionsItemId") tool_call_events: list[ToolCallEvent] = Field(default_factory=list, alias="toolCallEvents") reasoning_step_count: int = Field(default=0, alias="reasoningStepCount") reasoning_steps: list[str] = Field(default_factory=list, alias="reasoningSteps") diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index cef232a23..6f0876cdc 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -9,7 +9,14 @@ import httpx import pytest from gooddata_eval.core.agentic import _conversation_context as ctx_mod -from gooddata_eval.core.agentic._clarification import fetch_object_refs, read_first_reply, ungrounded_refs +from gooddata_eval.core.agentic._clarification import ( + ClarifyingPick, + after_answers_mismatches, + build_clarifying_answers, + fetch_object_refs, + read_first_reply, + ungrounded_refs, +) from gooddata_eval.core.agentic._conversation_context import ( CONFIRMATION_REPLY, NO_ANSWER_REPLY, @@ -1528,7 +1535,17 @@ def _run( ) -> tuple[ConversationResult, MagicMock]: client = MagicMock() client.create_conversation.return_value = "conv-1" - client.send_message.side_effect = replies + # Text messages and clarifying answers take their replies from the one queue, in call order. + queue = iter(replies) + + def _next_reply(*_args: object, **_kwargs: object) -> ChatResult: + reply = next(queue) + if isinstance(reply, BaseException): + raise reply + return reply + + client.send_message.side_effect = _next_reply + client.send_clarifying_answers.side_effect = _next_reply fixture = ConversationFixture(id="c", expected_skills=["visualization"], turns=turns) with ( patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=client) as client_cls, @@ -2634,3 +2651,370 @@ def handler(request: httpx.Request) -> httpx.Response: with pytest.raises(ValueError): fetch_object_refs("https://h/", "t", "ws", transport=httpx.MockTransport(handler)) + + +# Structured answers and what the agent does after a skip. + +_SALES_CARD = { + "type": "clarifyingQuestions", + "questions": [ + { + "text": "Which sales measure should the alert monitor?", + "control": { + "type": "singleChoice", + "options": [{"text": "{metric/total_sales}"}, {"text": "{metric/net_sales}"}], + }, + }, + { + "text": "Which notification channel should I use?", + "control": {"type": "singleChoice", "options": [{"text": "HN SMTP"}, {"text": "Default SMTP"}]}, + }, + ], +} +_SALES_VIZ = _viz(metric="metric/net_sales", dim="label/product_category") + + +def _card(part: dict, item_id: str = "q-item", text: str = "Two sales measures match.") -> ChatResult: + return _result(text=text, tools=[_SKILLS], unhandledParts=[part], clarifyingQuestionsItemId=item_id) + + +def _answers_turn(picks: list[dict], **kw: object) -> TurnDefinition: + return _viz_turn( + "t1", + expected=kw.pop("expected", _SALES_VIZ), + message="Show me sales by product category.", + expects_clarification=True, + clarifying_answers=picks, + **kw, + ) + + +def test_picks_are_sent_as_clarifying_answers_and_an_unpicked_question_is_skipped() -> None: + turn_def = _answers_turn([{"option": "metric/net_sales"}]) + result, client = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def]) + client.send_clarifying_answers.assert_called_once_with( + "conv-1", + "q-item", + [ + { + "questionIndex": 0, + "questionText": "Which sales measure should the alert monitor?", + "answer": {"type": "singleChoice", "optionIndex": 1, "optionText": "{metric/net_sales}"}, + } + ], + user_context=None, + ) + turn = result.turn_results[0] + assert turn.clarification_correct is True + assert turn.context_success is True + assert turn.clarifications[0].reply == ( + "**Which sales measure should the alert monitor?** {metric/net_sales} " + "**Which notification channel should I use?** Skipped" + ) + + +def test_a_pick_can_name_the_option_text_and_its_question() -> None: + turn_def = _answers_turn([{"option": "default smtp", "question": 1}], expected=_SALES_VIZ) + _, client = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def]) + answers = client.send_clarifying_answers.call_args.args[2] + assert [(a["questionIndex"], a["answer"]["optionIndex"]) for a in answers] == [(1, 1)] + + +def test_an_empty_pick_list_skips_every_question() -> None: + _, client = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [_answers_turn([])]) + assert client.send_clarifying_answers.call_args.args[2] == [] + + +def test_a_pick_no_questionnaire_offered_fails_the_turn() -> None: + result, _ = _run( + [_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [_answers_turn([{"option": "metric/order_amount"}])] + ) + turn = result.turn_results[0] + assert turn.clarification_correct is False + assert "no questionnaire offered the picked option 'metric/order_amount'" in turn.failure_reasons() + + +def test_a_question_without_a_questionnaire_gets_the_text_answer_and_fails_the_answers_check() -> None: + turn_def = _answers_turn([{"option": "metric/net_sales"}], set_answers=["Use Net Sales."]) + result, client = _run( + [_result(text="Which sales measure do you mean?", tools=[_SKILLS]), _result(viz=_SALES_VIZ)], [turn_def] + ) + client.send_clarifying_answers.assert_not_called() + assert client.send_message.call_args_list[1].args[1] == "Use Net Sales." + assert result.turn_results[0].clarification_mismatches == [ + "the turn checks a questionnaire, but the first reply asked as text", + "clarifying_answers not sent: the first reply asked no questionnaire to answer", + ] + + +def test_answers_not_sent_because_the_first_reply_also_delivered_the_output_say_so() -> None: + reply = _result( + text="Here is net sales.", + viz=_SALES_VIZ, + tools=[_SKILLS], + unhandledParts=[_SALES_CARD], + clarifyingQuestionsItemId="q-item", + ) + result, client = _run([reply], [_answers_turn([{"option": "metric/net_sales"}])]) + client.send_clarifying_answers.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "clarifying_answers not sent: the first reply delivered the output with its questionnaire, " + "so nothing waited on the answers" + ] + + +def test_answers_not_sent_because_the_budget_ended_the_turn_say_so() -> None: + result, client = _run( + [_card(_SALES_CARD)], [_answers_turn([{"option": "metric/net_sales"}])], max_clarification_turns=0 + ) + client.send_clarifying_answers.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "clarifying_answers not sent: the clarification budget (0) ended the turn before clarifying_answers were sent" + ] + + +def test_answers_not_sent_because_the_questionnaire_has_no_item_id_say_so() -> None: + no_id = _result(text="Two sales measures match.", tools=[_SKILLS], unhandledParts=[_SALES_CARD]) + turn_def = _answers_turn([{"option": "metric/net_sales"}], set_answers=["Use Net Sales."]) + result, client = _run([no_id, _result(viz=_SALES_VIZ)], [turn_def]) + client.send_clarifying_answers.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "clarifying_answers not sent: the first reply's questionnaire has no itemId to send clarifying_answers to" + ] + + +def test_a_lost_context_verdict_on_the_expected_questionnaire_does_not_divert_the_answers() -> None: + turn_def = _answers_turn([{"option": "metric/net_sales"}]) + result, client = _run( + [_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def], judge=_FakeJudge(lost_context=True) + ) + assert client.send_clarifying_answers.call_args.args[1] == "q-item" + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + ("final_text", "named"), + [ + ("You skipped the measure, so I used {metric/total_sales}.", True), + ("I used Default SMTP for the channel and net sales for the rest.", False), + ("Here is the chart.", False), + ], +) +def test_the_final_reply_must_name_an_option_of_each_skipped_question(final_text: str, named: bool) -> None: + card = {"type": "clarifyingQuestions", "questions": [_SALES_CARD["questions"][0]]} + turn_def = _answers_turn( + [], states_skipped_assumption=True, expected=_viz(metric="metric/total_sales", dim="label/product_category") + ) + result, _ = _run( + [_card(card), _result(text=final_text, viz=_viz(metric="metric/total_sales", dim="label/product_category"))], + [turn_def], + ) + turn = result.turn_results[0] + assert turn.clarification_correct is named + if not named: + assert turn.clarification_mismatches == [ + "no reply after the answers names an option of the skipped question " + "'Which sales measure should the alert monitor?'" + ] + + +def test_an_assumption_stated_in_the_reply_that_asks_for_confirmation_counts() -> None: + card = {"type": "clarifyingQuestions", "questions": [_SALES_CARD["questions"][0]]} + total_sales = _viz(metric="metric/total_sales", dim="label/product_category") + turn_def = _answers_turn( + [], + set_answers=["Yes, go ahead."], + states_skipped_assumption=True, + expects_confirmation_after_answers=True, + expected=total_sales, + ) + result, _ = _run( + [ + _card(card), + _result(text="I'll assume {metric/total_sales}, OK?", tools=[_SKILLS]), + _result(text="", viz=total_sales), + ], + [turn_def], + ) + assert result.turn_results[0].clarification_correct is True + + +def test_asking_a_skipped_question_again_fails_the_turn() -> None: + reworded = { + "type": "clarifyingQuestions", + "questions": [{"text": "Which measure did you mean?", "control": _SALES_CARD["questions"][0]["control"]}], + } + turn_def = _answers_turn([], skipped_questions_not_reasked=True) + result, client = _run([_card(_SALES_CARD), _card(reworded, item_id="q-2"), _result(viz=_SALES_VIZ)], [turn_def]) + assert client.send_clarifying_answers.call_count == 1 + assert result.turn_results[0].clarification_mismatches == [ + "asked the skipped question 'Which sales measure should the alert monitor?' again" + ] + + +def test_a_new_question_after_a_skip_is_not_a_reasked_one() -> None: + other = { + "type": "clarifyingQuestions", + "questions": [ + {"text": "Which dashboard?", "control": {"options": [{"text": "1. Overview"}, {"text": "2. Sales"}]}} + ], + } + turn_def = _answers_turn([], skipped_questions_not_reasked=True) + result, _ = _run([_card(_SALES_CARD), _card(other, item_id="q-2"), _result(viz=_SALES_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True + + +@pytest.mark.parametrize( + ("later_text", "reasked"), + [("Should I build the chart?", False), ("Include returns?", True)], +) +def test_yes_no_options_count_as_a_reask_only_with_the_same_text(later_text: str, reasked: bool) -> None: + later = { + "type": "clarifyingQuestions", + "questions": [{"text": later_text, "control": {"options": [{"text": "Yes"}, {"text": "No"}]}}], + } + mismatches = after_answers_mismatches( + unmatched_picks=[], + skipped=[{"index": 0, "text": "Include returns?", "options": ["No", "Yes"]}], + replies_after=[("question", _result(unhandledParts=[later]), True)], + states_skipped_assumption=False, + skipped_questions_not_reasked=True, + expects_confirmation_after_answers=False, + ) + assert mismatches == (["asked the skipped question 'Include returns?' again"] if reasked else []) + + +@pytest.mark.parametrize( + ("options", "final_text", "named"), + [ + (["All", "Top 10"], "Overall revenue grew.", False), + (["All", "Top 10"], "I kept all products.", True), + (["{metric/net_sales}", "{metric/total_sales}"], "I used {metric/net_sales_ytd}.", False), + (["{attribute/product_category}", "{attribute/region}"], "Split by {label/product_category}.", True), + ], +) +def test_a_skipped_option_is_named_only_as_whole_words_or_a_whole_reference( + options: list[str], final_text: str, named: bool +) -> None: + mismatches = after_answers_mismatches( + unmatched_picks=[], + skipped=[{"index": 0, "text": "Which one?", "options": options}], + replies_after=[("output", _result(text=final_text), True)], + states_skipped_assumption=True, + skipped_questions_not_reasked=False, + expects_confirmation_after_answers=False, + ) + assert (mismatches == []) is named + + +def test_a_pick_by_attribute_matches_an_option_written_as_its_label() -> None: + card = _questionnaire(["{label/product_category}", "{label/region}"]) + built = build_clarifying_answers(_card(card), [ClarifyingPick(option="attribute/product_category")]) + assert [(a["questionIndex"], a["answer"]["optionIndex"]) for a in built.answers] == [(0, 0)] + assert built.matched_picks == ["attribute/product_category"] + + +def test_answers_number_the_questions_of_the_item_they_answer_only() -> None: + first = {**_questionnaire(["{metric/returns}", "{metric/total_returns}"]), "itemId": "i-1"} + second = {**_questionnaire(["{metric/net_sales}", "{metric/total_sales}"]), "itemId": "i-2"} + reply = _result(unhandledParts=[first, second], clarifyingQuestionsItemId="i-2") + built = build_clarifying_answers(reply, [ClarifyingPick(option="metric/total_sales")]) + assert [(a["questionIndex"], a["answer"]["optionIndex"]) for a in built.answers] == [(0, 1)] + assert built.skipped == [] + + +def test_acting_on_the_answers_without_a_confirmation_fails_when_one_is_expected() -> None: + turn_def = _answers_turn([], expects_confirmation_after_answers=True) + result, _ = _run([_card(_SALES_CARD), _result(viz=_SALES_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_mismatches == [ + "acted on the answers in the same reply, without waiting for confirmation" + ] + + +def test_a_confirmation_before_the_output_passes_when_one_is_expected() -> None: + turn_def = _answers_turn([], expects_confirmation_after_answers=True) + result, client = _run( + [_card(_SALES_CARD), _result(text="I will chart Net Sales. Please confirm."), _result(viz=_SALES_VIZ)], + [turn_def], + ) + assert result.turn_results[0].clarification_correct is True + assert client.send_message.call_args_list[-1].args[1] == CONFIRMATION_REPLY + + +@pytest.mark.parametrize( + "keys", + [ + {"states_skipped_assumption": True}, + {"skipped_questions_not_reasked": True}, + {"expects_confirmation_after_answers": True}, + ], +) +def test_an_after_answers_key_without_clarifying_answers_is_rejected(keys: dict) -> None: + with pytest.raises(ValueError, match="set clarifying_answers"): + TurnDefinition(turn_id="t1", message="m", expected_skill="visualization", expects_clarification=True, **keys) + + +def test_clarifying_answers_need_an_expected_questionnaire() -> None: + with pytest.raises(ValueError, match="need expects_clarification: true"): + TurnDefinition(turn_id="t1", message="m", expected_skill="visualization", clarifying_answers=[]) + with pytest.raises(ValueError, match="not text"): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + expected_clarification_form="text", + clarifying_answers=[], + ) + + +def test_rendered_answers_follow_the_text_gen_ai_builds_for_the_model() -> None: + built = build_clarifying_answers(_card(_SALES_CARD), [ClarifyingPick(option="HN SMTP")]) + assert [s["index"] for s in built.skipped] == [0] + assert built.render() == ( + "**Which sales measure should the alert monitor?**\n\nSkipped\n\n" + "**Which notification channel should I use?**\n\nHN SMTP" + ) + + +def test_a_plan_the_agent_waits_on_after_the_answers_counts_as_stopping_for_confirmation() -> None: + turn_def = _answers_turn([], expects_confirmation_after_answers=True) + result, _ = _run( + [ + _card(_SALES_CARD), + _result(text="I'll use Net Sales. The plan: one bar chart by category."), + _result(viz=_SALES_VIZ), + ], + [turn_def], + ) + assert result.turn_results[0].clarification_mismatches == [] + + +def test_a_pick_names_the_spellings_of_one_option() -> None: + plain = { + "type": "clarifyingQuestions", + "questions": [ + {"text": "Which sales measure?", "control": {"options": [{"text": "Total Sales"}, {"text": "Net Sales"}]}} + ], + } + turn_def = _answers_turn([{"option": ["metric/net_sales", "Net Sales"]}]) + result, client = _run([_card(plain), _result(viz=_SALES_VIZ)], [turn_def]) + assert client.send_clarifying_answers.call_args.args[2][0]["answer"]["optionText"] == "Net Sales" + assert result.turn_results[0].clarification_correct is True + + +def test_only_the_first_questionnaire_gets_the_structured_answers() -> None: + confirm = { + "type": "clarifyingQuestions", + "questions": [{"text": "Should I build it?", "control": {"options": [{"text": "Yes"}, {"text": "No"}]}}], + } + turn_def = _answers_turn([], set_answers=["Yes, build it."], expects_confirmation_after_answers=True) + result, client = _run([_card(_SALES_CARD), _card(confirm, item_id="q-2"), _result(viz=_SALES_VIZ)], [turn_def]) + assert client.send_clarifying_answers.call_count == 1 + assert client.send_message.call_args_list[-1].args[1] == "Yes, build it." + assert result.turn_results[0].clarification_correct is True + + +def test_an_empty_option_spelling_is_rejected() -> None: + with pytest.raises(ValueError, match="non-empty spelling"): + ClarifyingPick(option=[]) diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 510f2b0ca..629585358 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -1422,3 +1422,49 @@ def test_parse_sse_lines_adhoc_fallback_keeps_its_id_over_one_in_the_arguments() result = parse_sse_lines(lines) assert result.created_visualizations is not None assert result.created_visualizations.objects[0].id == "adhoc-visualization-not-persisted" + + +def test_parse_sse_lines_keeps_the_item_id_of_the_questionnaire(): + card = {"type": "clarifyingQuestions", "questions": [{"text": "Which one?", "control": {"options": []}}]} + lines = [ + 'data: {"item": {"itemId": "i-text", "role": "assistant", "content": ' + '{"type": "multipart", "parts": [{"type": "text", "text": "Two measures."}]}}}', + "data: " + + json.dumps( + { + "item": { + "itemId": "i-card", + "role": "assistant", + "content": {"type": "multipart", "parts": [{"type": "text", "text": "Pick one."}, card]}, + } + } + ), + ] + result = parse_sse_lines(lines) + assert result.clarifying_questions_item_id == "i-card" + assert result.unhandled_parts == [{**card, "itemId": "i-card"}] + + +def test_parse_sse_lines_has_no_questionnaire_item_without_one(): + assert parse_sse_lines(_OK_SSE.decode().splitlines()).clarifying_questions_item_id is None + + +def test_send_clarifying_answers_sends_the_chat_panel_shape(): + captured = [] + answers = [ + { + "questionIndex": 0, + "questionText": "Which one?", + "answer": {"type": "singleChoice", "optionIndex": 1, "optionText": "{metric/net_sales}"}, + } + ] + _capture_body_client(captured, reasoning_effort="LOW").send_clarifying_answers("conv", "i-card", answers) + assert captured == [ + { + "item": { + "role": "user", + "content": {"type": "clarifyingAnswers", "questionsItemId": "i-card", "answers": answers}, + }, + "options": {"reasoningEffort": "LOW"}, + } + ] From 912ed953aa7f36e6603682a1249eb326e20b9e80 Mon Sep 17 00:00:00 2001 From: huyenthanh09 Date: Tue, 3 Nov 2026 16:50:00 +0700 Subject: [PATCH 3/3] feat(gooddata-eval): check plain-text clarifying options by title clarifying_options_grounded reads only {type/id} options, but gen-ai also writes options as plain names. Asked to describe a chart whose title two charts share, it offers names that point at no chart or at the wrong one: the second chart's description, which is a third chart's title, or a paraphrase like "Net Sales version". A user picking such an option gets another chart described. The workspace listing now keeps each object's title and each chart's type, and a turn gains two opt-in keys, both needing expects_clarification: true: - clarifying_option_titles: an object kind (visualization, metric, dashboard, attribute, fact, dataset). Every plain-text option must be the title of such an object, alone or followed by context in brackets or after a dash, comma or colon, as two charts with one title need. - clarifying_option_objects: the type/id objects the question is about. Every option must name one of them, by reference, by its title with that context, or by its chart type ("Donut chart") when no other candidate has that type and the option does not start with another object's title. A reference or title of any other object fails. Both read only the questions they are about: a question with at least one option the check accepts. A questionnaire that also asks for a channel or a breakdown no longer fails on those options. When no question is about the check, every option it reads fails. Grounding still reads every question. Like grounding, both leave the turn ungraded when the workspace cannot be read. The listing now goes through gooddata_sdk's EntitiesApi with load_all_entities, retrying a transient failure as a chat message is. It is read only when the first reply is a questionnaire with options, and kept for the run until the run creates or updates a metric or an alert. Checked live on an ecommerce_demo workspace with gpt-5.6-luna, asking to describe the Net Sales Breakdown chart, which a donut and a waterfall chart are titled: options naming invented versions fail, options naming the two charts by reference or by chart type pass. jira: QA-30039 risk: low Co-Authored-By: Claude Opus 5.5 --- .../core/agentic/_clarification.py | 273 ++++++++++-- .../core/agentic/conversation.py | 69 ++- .../tests/test_agentic_conversation.py | 403 ++++++++++++++---- 3 files changed, 611 insertions(+), 134 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py index 836d3a11e..86a23b375 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_clarification.py @@ -3,12 +3,18 @@ from __future__ import annotations +import functools import re +from collections.abc import Callable from typing import Any, Literal -import httpx +from gooddata_api_client.exceptions import ApiException, NotFoundException +from gooddata_sdk import GoodDataSdk +from gooddata_sdk.utils import load_all_entities from pydantic import BaseModel, Field, model_validator +from urllib3.exceptions import HTTPError +from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, retry_transient from gooddata_eval.core.models import ChatResult ClarificationForm = Literal["questionnaire", "text"] @@ -33,23 +39,77 @@ } # Reference types gen-ai writes that no entity listing holds, so grounding cannot check them. _UNLISTED_REF_TYPES = frozenset({"computed_attribute", "filter_context", "document", "report"}) -_GENERIC_ANSWERS = frozenset( - {"yes", "no", "ok", "okay", "cancel", "confirm", "continue", "proceed", "skip", "not now", "go ahead"} -) _ENTITY_KINDS = tuple(sorted({kind for kinds in _REF_ENTITIES.values() for kind in kinds})) _PAGE_SIZE = 1000 +# The EntitiesApi listing of each entity kind. +_LIST_METHODS: dict[str, str] = { + "analyticalDashboards": "get_all_entities_analytical_dashboards", + "attributes": "get_all_entities_attributes", + "datasets": "get_all_entities_datasets", + "facts": "get_all_entities_facts", + "labels": "get_all_entities_labels", + "metrics": "get_all_entities_metrics", + "parameters": "get_all_entities_parameters", + "visualizationObjects": "get_all_entities_visualization_objects", +} # A shorter skipped option, such as "No" or "Q1", also occurs in text that assumes nothing, so it # never counts as named. _MIN_NAMED_OPTION_CHARS = 3 +OptionTitleKind = Literal["metric", "attribute", "fact", "dataset", "dashboard", "visualization"] +_TITLE_ENTITIES: dict[str, str] = { + "metric": "metrics", + "attribute": "attributes", + "fact": "facts", + "dataset": "datasets", + "dashboard": "analyticalDashboards", + "visualization": "visualizationObjects", +} +# What may follow a title in an option that adds context, as two charts with one title need. +_TITLE_CONTEXT_SEPARATORS = (" (", " [", " - ", " – ", " — ", ": ", ", ") +_GENERIC_ANSWERS = frozenset( + {"yes", "no", "ok", "okay", "cancel", "confirm", "continue", "proceed", "skip", "not now", "go ahead"} +) +# How an option may spell a chart type whose `local:` id is not a word of its own. +_CHART_TYPE_WORDS: dict[str, tuple[str, ...]] = { + "combo2": ("combo",), + "pushpin": ("geo", "map", "pushpin"), + "heatmap": ("heatmap", "heat map"), + "treemap": ("treemap", "tree map"), + "dependencywheel": ("dependency wheel",), +} + + +class QuestionOptions(BaseModel): + """One question's options: the ``type/id`` references they name, and the options that name none.""" + + refs: list[str] = Field(default_factory=list) + plain: list[str] = Field(default_factory=list) + + class FirstReply(BaseModel): """What the turn's first assistant reply asked, read before any simulated-user answer.""" asked: bool form: ClarificationForm | None = None - question_count: int = 0 - option_refs: list[str] = Field(default_factory=list) + questions: list[QuestionOptions] = Field(default_factory=list) + + @property + def question_count(self) -> int: + return len(self.questions) + + @property + def option_refs(self) -> list[str]: + return [ref for q in self.questions for ref in q.refs] + + @property + def plain_options(self) -> list[str]: + return [option for q in self.questions for option in q.plain] + + @property + def has_options(self) -> bool: + return any(q.refs or q.plain for q in self.questions) def read_first_reply(chat_result: ChatResult, *, asked_in_text: bool) -> FirstReply: @@ -58,41 +118,76 @@ def read_first_reply(chat_result: ChatResult, *, asked_in_text: bool) -> FirstRe ``asked_in_text`` is the caller's reading of the prose, for a question without a part. """ parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"] - questions = [q for p in parts for q in p.get("questions") or []] - options = [str(o.get("text") or "") for q in questions for o in (q.get("control") or {}).get("options") or []] - refs = [f"{kind}/{obj_id}" for text in options for kind, obj_id in _REF_RE.findall(text)] - if parts: - return FirstReply(asked=True, form="questionnaire", question_count=len(questions), option_refs=refs) - return FirstReply(asked=asked_in_text, form="text" if asked_in_text else None) + if not parts: + return FirstReply(asked=asked_in_text, form="text" if asked_in_text else None) + questions = [] + for question in (q for p in parts for q in p.get("questions") or []): + options = _options(question) + questions.append( + QuestionOptions( + refs=[f"{kind}/{obj_id}" for text in options for kind, obj_id in _REF_RE.findall(text)], + plain=[text for text in options if not _REF_RE.search(text)], + ) + ) + return FirstReply(asked=True, form="questionnaire", questions=questions) + + +class WorkspaceObjects(BaseModel): + """The workspace's objects as the option checks read them: ``/`` and their titles.""" + + refs: set[str] = Field(default_factory=set) + # `/` -> the title, casefolded with whitespace collapsed. + titles: dict[str, str] = Field(default_factory=dict) + # `visualizationObjects/` -> the chart type, `local:donut` read as `donut`. + chart_types: dict[str, str] = Field(default_factory=dict) + + def titles_of(self, entity_kind: str) -> set[str]: + return {title for ref, title in self.titles.items() if ref.startswith(f"{entity_kind}/")} + + +def _title_key(text: str) -> str: + return " ".join(text.casefold().split()) + + +def _is_transient(exc: Exception) -> bool: + if isinstance(exc, ApiException): + return exc.status in RETRYABLE_STATUS_CODES + return isinstance(exc, HTTPError) + + +def _page_with_retry(list_page: Callable[..., Any], **page: int) -> Any: + return retry_transient(lambda: list_page(**page), is_retryable=_is_transient) -def fetch_object_refs( - host: str, token: str, workspace_id: str, *, transport: httpx.BaseTransport | None = None -) -> set[str]: - """Every ``/`` the workspace holds, inherited objects included. +def fetch_workspace_objects(sdk: GoodDataSdk, workspace_id: str) -> WorkspaceObjects: + """Every ``/`` and title the workspace holds, inherited objects included. - The entity API sends a ``next`` link on every page, an empty one too, so a short page ends the listing. - A kind the server does not know (404) has no objects. A body that is not an entity page raises ``ValueError``. + A page that fails with a transient error is retried as a chat message is. A kind the server does + not know (404) has no objects. Any other failure of the entity API is raised. """ - found: set[str] = set() - headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} - with httpx.Client(timeout=60, headers=headers, transport=transport) as client: - for kind in _ENTITY_KINDS: - url = f"{host.rstrip('/')}/api/v1/entities/workspaces/{workspace_id}/{kind}" - page = 0 - while True: - response = client.get(url, params={"size": _PAGE_SIZE, "page": page}) - if response.status_code == httpx.codes.NOT_FOUND: - break - response.raise_for_status() - body: Any = response.json() - data = body.get("data") if isinstance(body, dict) else None - if not isinstance(data, list): - raise ValueError(f"{kind} page {page} is not an entity page") - found.update(f"{kind}/{obj['id']}" for obj in data if isinstance(obj, dict) and obj.get("id")) - if len(data) < _PAGE_SIZE: - break - page += 1 + found = WorkspaceObjects() + api = sdk._client.entities_api + for kind in _ENTITY_KINDS: + list_page = functools.partial(getattr(api, _LIST_METHODS[kind]), workspace_id, _check_return_type=False) + try: + data = load_all_entities(functools.partial(_page_with_retry, list_page), page_size=_PAGE_SIZE).data + except NotFoundException: + continue + for obj in data: + if not isinstance(obj, dict) or not obj.get("id"): + continue + ref = f"{kind}/{obj['id']}" + found.refs.add(ref) + raw_attributes = obj.get("attributes") + attributes: dict[str, Any] = raw_attributes if isinstance(raw_attributes, dict) else {} + title = attributes.get("title") + if isinstance(title, str) and title: + found.titles[ref] = _title_key(title) + raw_content = attributes.get("content") + content: dict[str, Any] = raw_content if isinstance(raw_content, dict) else {} + chart_url = str(content.get("visualizationUrl") or "") + if chart_url.startswith("local:"): + found.chart_types[ref] = chart_url.removeprefix("local:") return found @@ -112,19 +207,100 @@ def ungrounded_refs(refs: list[str], known: set[str]) -> list[str]: return missing +def _is_titled(option: str, titles: set[str]) -> bool: + key = _title_key(option) + return key in titles or any(key.startswith(title + sep) for title in titles for sep in _TITLE_CONTEXT_SEPARATORS) + + +def untitled_options(options: list[str], objects: WorkspaceObjects, kind: OptionTitleKind) -> list[str]: + """The options that are not the title of a ``kind`` object, alone or followed by added context.""" + titles = objects.titles_of(_TITLE_ENTITIES[kind]) + return [option for option in options if not _is_titled(option, titles)] + + +def _off_in_scope( + questions: list[QuestionOptions], off_of: Callable[[QuestionOptions], tuple[int, list[str]]] +) -> list[str]: + """The options an option check fails, read only from the questions the check is about. + + ``off_of`` gives a question's count of options the check reads and the ones it fails. A questionnaire + can ask about more than one thing, such as a metric next to a channel: a question with at least one + option the check accepts is about it, and a question with none is about something else and left out. + When no question is about it, every option the check read fails. + """ + checked = [off_of(q) for q in questions] + about = [off for read, off in checked if len(off) < read] + return [option for off in (about or [off for _, off in checked]) for option in off] + + +def untitled_question_options(reply: FirstReply, objects: WorkspaceObjects, kind: OptionTitleKind) -> list[str]: + """The plain-text options that are not the title of a ``kind`` object, in the questions about that kind.""" + return _off_in_scope(reply.questions, lambda q: (len(q.plain), untitled_options(q.plain, objects, kind))) + + +def _entity_refs(ref: str) -> set[str]: + ref_type, _, obj_id = ref.partition("/") + return {f"{kind}/{obj_id}" for kind in _REF_ENTITIES.get(ref_type, ())} + + +def is_object_ref(ref: str) -> bool: + """Whether ``ref`` is ``type/id`` with a type an option reference can name.""" + ref_type, sep, obj_id = ref.partition("/") + return bool(sep and obj_id) and ref_type in _REF_ENTITIES + + +def _names_chart_type(option: str, chart_type: str) -> bool: + key = _title_key(option) + return any(re.search(rf"\b{re.escape(word)}\b", key) for word in _CHART_TYPE_WORDS.get(chart_type, (chart_type,))) + + +def off_candidate_options(reply: FirstReply, objects: WorkspaceObjects, candidates: list[str]) -> list[str]: + """The options that name none of ``candidates``, in the questions that offer at least one of them. + + An option names a candidate by reference, by its title plus context, or by its chart type when + no other candidate has that type: "Donut chart" tells two same-titled charts apart only then. + A chart type does not count in an option that starts with the title of an object outside the candidates. + """ + wanted = set().union(*(_entity_refs(c) for c in candidates)) + titles = {objects.titles[ref] for ref in wanted if ref in objects.titles} + kinds = {ref.partition("/")[0] for ref in wanted} + other_titles = { + title for ref, title in objects.titles.items() if ref not in wanted and ref.partition("/")[0] in kinds + } - titles + types = [objects.chart_types[ref] for ref in wanted if ref in objects.chart_types] + unique_types = {chart_type for chart_type in types if types.count(chart_type) == 1} + + def names_a_candidate_type(option: str) -> bool: + return any(_names_chart_type(option, t) for t in unique_types) and not _is_titled(option, other_titles) + + def off_of(question: QuestionOptions) -> tuple[int, list[str]]: + off = [ref for ref in question.refs if not (_entity_refs(ref) & wanted)] + off += [ + option for option in question.plain if not _is_titled(option, titles) and not names_a_candidate_type(option) + ] + return len(question.refs) + len(question.plain), off + + return _off_in_scope(reply.questions, off_of) + + def clarification_mismatches( reply: FirstReply, *, expects_clarification: bool, expected_form: ClarificationForm | None, max_questions: int | None, - known_refs: set[str] | None, + objects: WorkspaceObjects | None, + grounded: bool, + titled_as: OptionTitleKind | None, + candidates: list[str] | None = None, reads_questionnaire: bool = False, ) -> list[str]: """How the first reply differs from the expected clarification; empty when it matches. - ``known_refs`` is None when the fixture does not ask for grounded options. - Options without a ``{type/id}`` reference are not checked for grounding. + ``grounded`` checks the ``{type/id}`` options of every question; ``titled_as`` checks the plain-text + ones against that kind's titles; ``candidates`` requires every option to name one of those objects. + The last two read only the questions they are about (see ``_off_in_scope``). All three are skipped + when ``objects`` is None: the caller reports the turn ungraded then. ``reads_questionnaire`` fails a prose question, which has no options for these checks to read. """ if not expects_clarification: @@ -138,10 +314,18 @@ def clarification_mismatches( out.append(f"the turn checks a questionnaire, but the first reply asked as {reply.form}") if max_questions is not None and reply.question_count > max_questions: out.append(f"asked {reply.question_count} questions, at most {max_questions} expected") - if known_refs is not None: - missing = ungrounded_refs(reply.option_refs, known_refs) + if objects is not None and grounded: + missing = ungrounded_refs(reply.option_refs, objects.refs) if missing: out.append(f"options name objects the workspace does not have: {missing}") + if objects is not None and titled_as is not None: + untitled = untitled_question_options(reply, objects, titled_as) + if untitled: + out.append(f"options are not the title of a {titled_as} of the workspace: {untitled}") + if objects is not None and candidates: + off = off_candidate_options(reply, objects, candidates) + if off: + out.append(f"options name none of {candidates}: {off}") return out @@ -186,11 +370,6 @@ def render(self) -> str: return "\n\n".join(f"**{text}**\n\n{answer}" for _, text, answer in sorted(sections, key=lambda s: s[0])) -def _entity_refs(ref: str) -> set[str]: - ref_type, _, obj_id = ref.partition("/") - return {f"{kind}/{obj_id}" for kind in _REF_ENTITIES.get(ref_type, ())} - - def _all_questions(chat_result: ChatResult) -> list[dict]: parts = [p for p in chat_result.unhandled_parts or [] if p.get("type") == "clarifyingQuestions"] return [q for p in parts for q in p.get("questions") or []] diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 01eeb6c2d..e980958bc 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -31,10 +31,13 @@ ClarificationForm, ClarifyingPick, FirstReply, + OptionTitleKind, + WorkspaceObjects, after_answers_mismatches, build_clarifying_answers, clarification_mismatches, - fetch_object_refs, + fetch_workspace_objects, + is_object_ref, read_first_reply, ) from gooddata_eval.core.agentic._conversation_context import ( @@ -125,7 +128,12 @@ class TurnDefinition(BaseModel): it must not. Left out, nothing is checked. With true, ``expected_clarification_form`` names how it asks (``questionnaire`` = a ``clarifyingQuestions`` part, ``text`` = prose only), ``max_clarifying_questions`` caps the questionnaire, and ``clarifying_options_grounded`` - requires every ``{type/id}`` option to name an object of the workspace. + requires every ``{type/id}`` option to name an object of the workspace. ``clarifying_option_titles`` + names an object kind (``visualization``, ``metric``, ...): every plain-text option must be the title + of such an object, alone or followed by context in brackets or after a dash, comma or colon. + ``clarifying_option_objects`` lists the ``type/id`` objects the question is about, such as the + charts sharing the title the user named: every option must name one of them, by reference, by + its title with that context, or by its chart type when no other candidate has that type. ``clarifying_answers`` answers the questionnaire of the turn's first reply the way the chat panel does, with a ``clarifyingAnswers`` message: a question no pick matches is skipped, and an empty @@ -153,6 +161,8 @@ class TurnDefinition(BaseModel): expected_clarification_form: ClarificationForm | None = None max_clarifying_questions: int | None = Field(default=None, ge=1) clarifying_options_grounded: bool = False + clarifying_option_titles: OptionTitleKind | None = None + clarifying_option_objects: list[str] | None = None clarifying_answers: list[ClarifyingPick] | None = None states_skipped_assumption: bool = False skipped_questions_not_reasked: bool = False @@ -164,6 +174,8 @@ def reads_questionnaire(self) -> bool: return ( self.max_clarifying_questions is not None or self.clarifying_options_grounded + or self.clarifying_option_titles is not None + or self.clarifying_option_objects is not None or self.clarifying_answers is not None ) @@ -176,6 +188,8 @@ def _clarification_keys_need_an_expected_question(self) -> TurnDefinition: ("expected_clarification_form", self.expected_clarification_form), ("max_clarifying_questions", self.max_clarifying_questions), ("clarifying_options_grounded", self.clarifying_options_grounded or None), + ("clarifying_option_titles", self.clarifying_option_titles), + ("clarifying_option_objects", self.clarifying_option_objects), ("clarifying_answers", self.clarifying_answers), ) if value is not None @@ -184,8 +198,8 @@ def _clarification_keys_need_an_expected_question(self) -> TurnDefinition: raise ValueError(f"{stated} need expects_clarification: true") if self.reads_questionnaire and self.expected_clarification_form == "text": raise ValueError( - "max_clarifying_questions, clarifying_options_grounded and clarifying_answers read a " - "questionnaire, not text" + "max_clarifying_questions, clarifying_options_grounded, clarifying_option_titles, " + "clarifying_option_objects and clarifying_answers read a questionnaire, not text" ) after_answers = [ name @@ -196,6 +210,10 @@ def _clarification_keys_need_an_expected_question(self) -> TurnDefinition: ) if value ] + if self.clarifying_option_objects is not None: + bad = [ref for ref in self.clarifying_option_objects if not is_object_ref(ref)] + if not self.clarifying_option_objects or bad: + raise ValueError(f"clarifying_option_objects needs type/id references, got {bad or 'none'}") if after_answers and self.clarifying_answers is None: raise ValueError(f"{after_answers} read the turn after its clarifying_answers; set clarifying_answers") return self @@ -767,6 +785,10 @@ def __init__(self) -> None: self.updated_metrics: list[str] = [] self.alerts: list[str] = [] + def state(self) -> tuple[tuple[str, ...], ...]: + """What the conversation has written so far; a change means a workspace listing is out of date.""" + return tuple(self.metrics), tuple(self.updated_metrics), tuple(self.alerts) + def record(self, tool_call_events: list[ToolCallEvent]) -> None: for metric_id, created_new in _metric_creations(tool_call_events): if created_new: @@ -894,6 +916,8 @@ def run_agentic_conversation( run_started = time.monotonic() conversation_swap_s = 0.0 # fresh_conversation_per_turn's delete/create, which is not the run's work workspace_read_s = 0.0 # the option checks' workspace listing, which is not the run's work either + # The option checks' listing, with the created.state() it was read at; read again once that changes. + workspace_listing: tuple[tuple[tuple[str, ...], ...], WorkspaceObjects] | None = None for turn_index, turn in enumerate(fixture.turns): if fresh_conversation_per_turn and turn_index > 0: swap_started = time.monotonic() @@ -1157,22 +1181,37 @@ def run_agentic_conversation( clarification_problems: list[str] = [] clarification_ungraded: str | None = None if turn.expects_clarification is not None and first_reply is not None: - known_refs: set[str] | None = None - # Read on each such turn: an option can name an object an earlier turn created. - if turn.clarifying_options_grounded: - read_started = time.monotonic() - try: - known_refs = fetch_object_refs(host, token, workspace_id) or None - except Exception as exc: # any failure leaves the options unchecked, never passed - clarification_ungraded = f"workspace objects not readable, options left unchecked: {exc}" - print(f"[CLARIFICATION] {clarification_ungraded}") - workspace_read_s += time.monotonic() - read_started + checks_options = ( + turn.clarifying_options_grounded + or turn.clarifying_option_titles is not None + or turn.clarifying_option_objects is not None + ) + workspace_objects: WorkspaceObjects | None = None + # Only a questionnaire with options has anything to check. The listing is kept for the run + # and read again once a turn has written to the workspace, since an option can name an + # object an earlier turn created. + if checks_options and first_reply.has_options: + written = created.state() + if workspace_listing is None or workspace_listing[0] != written: + read_started = time.monotonic() + try: + workspace_listing = (written, fetch_workspace_objects(sdk, workspace_id)) + except Exception as exc: # any failure leaves the options unchecked, never passed + workspace_listing = None + clarification_ungraded = f"workspace objects not readable, options left unchecked: {exc}" + print(f"[CLARIFICATION] {clarification_ungraded}") + workspace_read_s += time.monotonic() - read_started + if workspace_listing is not None: + workspace_objects = workspace_listing[1] clarification_problems = clarification_mismatches( first_reply, expects_clarification=turn.expects_clarification, expected_form=turn.expected_clarification_form, max_questions=turn.max_clarifying_questions, - known_refs=known_refs, + objects=workspace_objects, + grounded=turn.clarifying_options_grounded, + titled_as=turn.clarifying_option_titles, + candidates=turn.clarifying_option_objects, reads_questionnaire=turn.reads_questionnaire, ) if turn.clarifying_answers is not None and not answers_sent: diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index 6f0876cdc..ca0f2dc2d 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -4,18 +4,23 @@ import sys import types from datetime import date +from types import SimpleNamespace from unittest.mock import MagicMock, patch -import httpx import pytest +from gooddata_api_client.exceptions import ApiException, NotFoundException from gooddata_eval.core.agentic import _conversation_context as ctx_mod from gooddata_eval.core.agentic._clarification import ( ClarifyingPick, + QuestionOptions, + WorkspaceObjects, after_answers_mismatches, build_clarifying_answers, - fetch_object_refs, + fetch_workspace_objects, + off_candidate_options, read_first_reply, ungrounded_refs, + untitled_options, ) from gooddata_eval.core.agentic._conversation_context import ( CONFIRMATION_REPLY, @@ -52,6 +57,7 @@ from gooddata_eval.core.chat.sse_client import ChatError as ChatError_ from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent +from urllib3.exceptions import ProtocolError def _skills_tc(*skills): @@ -2390,27 +2396,6 @@ def test_a_prose_question_passes_an_expected_text_question() -> None: assert result.turn_results[0].clarification_correct is True -@pytest.mark.parametrize( - ("text", "asks"), - [ - ("Did you mean Net Sales or Gross Sales? Please confirm.", True), - ("Which time range do you want?", True), - ("I will chart Total Returns. Please confirm.", False), - ("Should I proceed with Net Sales?", False), - ("Here is the plan. Shall I create it?", False), - ], -) -def test_a_question_followed_by_a_confirmation_phrase_still_asks(text: str, asks: bool) -> None: - assert asks_in_text(_result(text=text), text) is asks - - -def test_a_prose_choice_with_please_confirm_passes_an_expected_text_question() -> None: - turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="text") - text = "Did you mean Returns or Total Returns? Please confirm." - result, _ = _run([_result(text=text, tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def]) - assert result.turn_results[0].clarification_correct is True - - def test_more_questions_than_allowed_fail_the_turn() -> None: card = _questionnaire(["{metric/returns}", "{metric/total_returns}"], ["Monthly", "Quarterly"]) turn_def = _clarifying_turn(expects_clarification=True, max_clarifying_questions=1) @@ -2420,17 +2405,22 @@ def test_more_questions_than_allowed_fail_the_turn() -> None: def test_an_option_naming_an_object_the_workspace_lacks_fails_grounding() -> None: card = _questionnaire(["{metric/total_returns}", "{dashboard/2. Sales}", "Plain text option"]) - known = {"metrics/total_returns", "analyticalDashboards/bf439696"} + known = WorkspaceObjects(refs={"metrics/total_returns", "analyticalDashboards/bf439696"}) turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) - with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", return_value=known) as fetch: + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known) as fetch: result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) - fetch.assert_called_once_with("h", "t", "ws") + assert fetch.call_count == 1 + assert fetch.call_args.args[1] == "ws" assert result.turn_results[0].clarification_mismatches == [ "options name objects the workspace does not have: ['dashboard/2. Sales']" ] -def test_grounded_options_pass_and_the_catalog_is_read_on_each_turn_that_checks_options() -> None: +def _created_metric(metric_id: str) -> dict: + return _tool("create_metric", result={"metric_id": metric_id, "created_new": True}) + + +def test_grounded_options_pass_and_the_listing_is_read_once_while_the_run_writes_nothing() -> None: turns = [ _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True), _viz_turn( @@ -2442,8 +2432,8 @@ def test_grounded_options_pass_and_the_catalog_is_read_on_each_turn_that_checks_ clarifying_options_grounded=True, ), ] - known = {"metrics/returns", "metrics/total_returns"} - with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", return_value=known) as fetch: + known = WorkspaceObjects(refs={"metrics/returns", "metrics/total_returns"}) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known) as fetch: result, _ = _run( [ _card_reply(_RETURNS_CARD), @@ -2453,14 +2443,57 @@ def test_grounded_options_pass_and_the_catalog_is_read_on_each_turn_that_checks_ ], turns, ) - assert fetch.call_count == 2 + assert fetch.call_count == 1 assert [t.clarification_correct for t in result.turn_results] == [True, True] -@pytest.mark.parametrize("error", [httpx.ConnectError("refused"), ValueError("metrics page 0 is not an entity page")]) +def test_the_listing_is_read_again_once_the_run_has_created_an_object() -> None: + turns = [ + _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True), + _viz_turn( + "t2", + expected=_RETURNS_VIZ, + message="Again for returns.", + set_answers=["Use Total Returns."], + expects_clarification=True, + clarifying_options_grounded=True, + ), + ] + known = WorkspaceObjects(refs={"metrics/returns", "metrics/total_returns"}) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known) as fetch: + _run( + [ + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ), + _card_reply(_RETURNS_CARD), + _result(viz=_RETURNS_VIZ, tools=[_created_metric("return_rate")]), + ], + turns, + ) + assert fetch.call_count == 2 + + +def test_the_listing_is_not_read_for_a_first_reply_without_options() -> None: + turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects") as fetch: + result, _ = _run( + [_result(text="Which returns measure do you want?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], + [turn_def], + ) + fetch.assert_not_called() + assert result.turn_results[0].clarification_mismatches == [ + "the turn checks a questionnaire, but the first reply asked as text" + ] + + +@pytest.mark.parametrize( + "error", + [ApiException(status=401, reason="Unauthorized"), ProtocolError("connection reset"), ValueError("x")], + ids=["unauthorized", "connection-reset", "unexpected-body"], +) def test_an_unreadable_listing_leaves_the_turn_ungraded_rather_than_passed(error: Exception) -> None: turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) - with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", side_effect=error): + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", side_effect=error): result, _ = _run([_card_reply(_RETURNS_CARD), _result(viz=_RETURNS_VIZ)], [turn_def]) turn = result.turn_results[0] assert turn.clarification_correct is None @@ -2476,7 +2509,7 @@ def test_an_unreadable_listing_still_fails_a_check_that_does_not_need_it() -> No expects_clarification=True, clarifying_options_grounded=True, max_clarifying_questions=1 ) card = _questionnaire(["{metric/returns}", "{metric/total_returns}"], ["Monthly", "Quarterly"]) - with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", side_effect=ValueError("x")): + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", side_effect=ValueError("x")): result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) assert result.turn_results[0].clarification_correct is False @@ -2501,18 +2534,45 @@ def test_ungrounded_refs_accept_each_spelling_of_an_object(ref: str, known: set[ assert (ungrounded_refs([ref], known) == []) is grounded -def test_read_first_reply_counts_questions_and_collects_option_refs() -> None: +def test_read_first_reply_keeps_the_options_of_each_question_apart() -> None: card = _questionnaire(["{metric/returns}", "Returned amount {metric/total_returns}"], ["Monthly", "Weekly"]) reply = read_first_reply(_result(unhandledParts=[card]), asked_in_text=False) assert reply.form == "questionnaire" assert reply.question_count == 2 + assert reply.questions == [ + QuestionOptions(refs=["metric/returns", "metric/total_returns"]), + QuestionOptions(plain=["Monthly", "Weekly"]), + ] assert reply.option_refs == ["metric/returns", "metric/total_returns"] + assert reply.plain_options == ["Monthly", "Weekly"] + + +@pytest.mark.parametrize( + ("text", "asks"), + [ + ("Did you mean Net Sales or Gross Sales? Please confirm.", True), + ("Which time range do you want?", True), + ("I will chart Total Returns. Please confirm.", False), + ("Should I proceed with Net Sales?", False), + ("Here is the plan. Shall I create it?", False), + ], +) +def test_a_question_followed_by_a_confirmation_phrase_still_asks(text: str, asks: bool) -> None: + assert asks_in_text(_result(text=text), text) is asks + + +def test_a_prose_choice_with_please_confirm_passes_an_expected_text_question() -> None: + turn_def = _clarifying_turn(expects_clarification=True, expected_clarification_form="text") + text = "Did you mean Returns or Total Returns? Please confirm." + result, _ = _run([_result(text=text, tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True def test_read_first_reply_reads_a_reference_type_with_an_underscore() -> None: card = _questionnaire(["{date_attribute/order_date.month}", "{date_attribute/ship_date.month}"]) reply = read_first_reply(_result(unhandledParts=[card]), asked_in_text=False) assert reply.option_refs == ["date_attribute/order_date.month", "date_attribute/ship_date.month"] + assert reply.plain_options == [] def test_clarification_checks_in_legacy_mode_are_refused_before_any_request() -> None: @@ -2521,8 +2581,9 @@ def test_clarification_checks_in_legacy_mode_are_refused_before_any_request() -> def test_option_keys_fail_a_question_asked_in_prose() -> None: - turn_def = _clarifying_turn(expects_clarification=True, clarifying_options_grounded=True) - with patch("gooddata_eval.core.agentic.conversation.fetch_object_refs", return_value={"metrics/total_returns"}): + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_objects=["metric/total_returns"]) + known = WorkspaceObjects(refs={"metrics/total_returns"}) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known): result, _ = _run( [_result(text="Do you mean returns or total returns?", tools=[_SKILLS]), _result(viz=_RETURNS_VIZ)], [turn_def], @@ -2608,49 +2669,122 @@ def _capture(_submit: object, _identity: object, **kwargs: object) -> None: assert scores["turn_t1_clarification_correct"] == 1.0 -def test_fetch_object_refs_ends_on_a_short_page_although_every_page_links_a_next_one( - monkeypatch: pytest.MonkeyPatch, -) -> None: +def _entities_sdk(pages: dict[tuple[str, int], list | Exception]) -> MagicMock: + """An sdk whose EntitiesApi serves ``pages``: the objects, or an exception to raise, per (kind, page).""" + sdk = MagicMock() + calls: list[tuple[str, int]] = [] + + def method(name: str) -> object: + def list_page(workspace_id: str, *, page: int, size: int, _check_return_type: bool) -> SimpleNamespace: + assert (workspace_id, _check_return_type) == ("ws", False) + calls.append((name, page)) + served = pages.get((name, page), []) + if isinstance(served, Exception): + raise served + return SimpleNamespace(data=served, included=[]) + + return list_page + + sdk._client.entities_api.configure_mock( + **{ + f"get_all_entities_{kind}": method(kind) + for kind in ( + "analytical_dashboards", + "attributes", + "datasets", + "facts", + "labels", + "metrics", + "parameters", + "visualization_objects", + ) + } + ) + sdk.calls = calls + return sdk + + +def test_fetch_workspace_objects_pages_each_kind_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("gooddata_eval.core.agentic._clarification._PAGE_SIZE", 2) - pages = {("metrics", 0): ["a", "b"], ("metrics", 1): ["c"]} - seen: list[tuple[str, int]] = [] - - def handler(request: httpx.Request) -> httpx.Response: - kind = request.url.path.rsplit("/", 1)[1] - page = int(request.url.params["page"]) - seen.append((kind, page)) - if kind == "parameters": - return httpx.Response(404, json={"title": "Not Found"}) - assert request.headers["Authorization"] == "Bearer t" - ids = pages.get((kind, page), []) - nxt = f"{request.url.copy_with(params={'page': page + 1, 'size': 2})}" - data: list = [{"id": i} for i in ids] - if (kind, page) == ("metrics", 0): - data += ["junk", {"attributes": {"title": "No id"}}] - return httpx.Response(200, json={"data": data, "links": {"next": nxt}}) - - refs = fetch_object_refs("https://h/", "t", "ws", transport=httpx.MockTransport(handler)) - assert {r for r in refs if r.startswith("metrics/")} == {"metrics/a", "metrics/b", "metrics/c"} - assert [p for k, p in seen if k == "metrics"] == [0, 1] - assert max(p for _, p in seen) == 1 - assert not any(r.startswith("parameters/") for r in refs) + metric = lambda i: {"id": i, "attributes": {"title": f"Metric {i.upper()}"}} # noqa: E731 + sdk = _entities_sdk( + { + ("metrics", 0): [metric("a"), metric("b"), "junk", {"attributes": {"title": "No id"}}], + ("metrics", 1): [metric("c")], + ("parameters", 0): NotFoundException(status=404, reason="Not Found"), + ("visualization_objects", 0): [ + {"id": "v", "attributes": {"title": "V", "content": {"visualizationUrl": "local:donut"}}} + ], + } + ) + objects = fetch_workspace_objects(sdk, "ws") + assert {r for r in objects.refs if r.startswith("metrics/")} == {"metrics/a", "metrics/b", "metrics/c"} + assert objects.titles_of("metrics") == {"metric a", "metric b", "metric c"} + assert objects.chart_types == {"visualizationObjects/v": "donut"} + assert [p for k, p in sdk.calls if k == "metrics"] == [0, 1] + assert not any(r.startswith("parameters/") for r in objects.refs) + + +def test_fetch_workspace_objects_retries_a_transient_failure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", "0") + sdk = _entities_sdk({("metrics", 0): [{"id": "a", "attributes": {"title": "A"}}]}) + list_metrics = sdk._client.entities_api.get_all_entities_metrics + failures = [ApiException(status=503, reason="Service Unavailable")] + + def flaky(workspace_id: str, **kw: object) -> SimpleNamespace: + if failures: + raise failures.pop() + return list_metrics(workspace_id, **kw) + + sdk._client.entities_api.get_all_entities_metrics = flaky + assert "metrics/a" in fetch_workspace_objects(sdk, "ws").refs + assert not failures + + +def test_fetch_workspace_objects_raises_a_failure_that_is_not_transient() -> None: + sdk = _entities_sdk({("metrics", 0): ApiException(status=401, reason="Unauthorized")}) + with pytest.raises(ApiException): + fetch_workspace_objects(sdk, "ws") + + +# Option checks read only the questions they are about. + + +def test_an_option_check_ignores_a_question_about_something_else() -> None: + # Reproduced on ecommerce_demo with gpt-5.6-luna: one questionnaire asking for the measure and the breakdown. + card = _questionnaire( + ["{metric/total_sales}", "{metric/net_sales}"], + ["{attribute/product_category}", "{attribute/product_brand}"], + ) + known = WorkspaceObjects( + refs={"metrics/total_sales", "metrics/net_sales", "attributes/product_category", "attributes/product_brand"} + ) + turn_def = _clarifying_turn( + expects_clarification=True, + clarifying_options_grounded=True, + clarifying_option_objects=["metric/total_sales", "metric/net_sales"], + ) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=known): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + assert result.turn_results[0].clarification_correct is True @pytest.mark.parametrize( - "response", + ("questions", "untitled"), [ - httpx.Response(200, text="Sign in"), - httpx.Response(200, json=[{"id": "a"}]), - httpx.Response(200, json={"errors": ["boom"]}), + ([["Net Sales", "Total Sales"], ["Email", "Slack"], ["2. Sales", "1. Overview"]], []), + ([["Net Sales", "Gross Margin"], ["Email", "Slack"]], ["Gross Margin"]), + ([["Email", "Slack"]], ["Email", "Slack"]), ], - ids=["not-json", "a-list", "no-data"], + ids=["other-questions-left-out", "off-option-in-the-metric-question", "no-question-about-metrics"], ) -def test_fetch_object_refs_refuses_a_body_that_is_not_an_entity_page(response: httpx.Response) -> None: - def handler(request: httpx.Request) -> httpx.Response: - return response - - with pytest.raises(ValueError): - fetch_object_refs("https://h/", "t", "ws", transport=httpx.MockTransport(handler)) +def test_a_title_check_reads_the_questions_about_its_kind(questions: list[list[str]], untitled: list[str]) -> None: + titles = WorkspaceObjects(titles={"metrics/net_sales": "net sales", "metrics/total_sales": "total sales"}) + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_titles="metric") + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=titles): + result, _ = _run([_card_reply(_questionnaire(*questions)), _result(viz=_RETURNS_VIZ)], [turn_def]) + expected = [f"options are not the title of a metric of the workspace: {untitled}"] if untitled else [] + assert result.turn_results[0].clarification_mismatches == expected # Structured answers and what the agent does after a skip. @@ -3018,3 +3152,128 @@ def test_only_the_first_questionnaire_gets_the_structured_answers() -> None: def test_an_empty_option_spelling_is_rejected() -> None: with pytest.raises(ValueError, match="non-empty spelling"): ClarifyingPick(option=[]) + + +# Plain-text options checked against the titles of one object kind (search: B15). + +# Two charts share one title, and the second one's description is another chart's title (ecommerce_demo). +_NSB_1, _NSB_2, _SALES_GOAL = ( + "visualizationObjects/643699f3", + "visualizationObjects/68ddf963", + "visualizationObjects/8a07a527", +) +_CHART_TITLES = WorkspaceObjects( + refs={_NSB_1, _NSB_2, _SALES_GOAL, "visualizationObjects/700d", "visualizationObjects/8619", "metrics/net_sales"}, + titles={ + _NSB_1: "net sales breakdown", + _NSB_2: "net sales breakdown", + _SALES_GOAL: "sales goal", + "visualizationObjects/700d": "net sales", + "visualizationObjects/8619": "total sales", + "metrics/net_sales": "net sales", + }, + chart_types={_NSB_1: "donut", _NSB_2: "waterfall", _SALES_GOAL: "bullet", "visualizationObjects/700d": "donut"}, +) + + +@pytest.mark.parametrize( + ("options", "untitled"), + [ + (["Net Sales Breakdown (1. Overview)", "Net Sales Breakdown (2. Sales)"], []), + (["Net Sales Breakdown - by tab", "net sales"], []), + (["Net sales split", "Net Sales by Region chart"], ["Net sales split", "Net Sales by Region chart"]), + ], +) +def test_plain_options_must_be_titles_of_the_named_kind(options: list[str], untitled: list[str]) -> None: + card = { + "type": "clarifyingQuestions", + "questions": [{"text": "Which chart?", "control": {"options": [{"text": o} for o in options]}}], + } + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_titles="visualization") + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=_CHART_TITLES): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + expected = [f"options are not the title of a visualization of the workspace: {untitled}"] if untitled else [] + assert result.turn_results[0].clarification_mismatches == expected + + +def test_title_check_reads_only_options_without_a_reference_and_only_the_named_kind() -> None: + reply = read_first_reply( + _result(unhandledParts=[_questionnaire(["{visualization/abc}", "Total Sales", "Gross Profit Trend"])]), + asked_in_text=False, + ) + assert reply.plain_options == ["Total Sales", "Gross Profit Trend"] + assert untitled_options(reply.plain_options, _CHART_TITLES, "visualization") == ["Gross Profit Trend"] + assert untitled_options(["Total Sales"], _CHART_TITLES, "metric") == ["Total Sales"] + + +def test_a_title_kind_needs_an_expected_questionnaire() -> None: + with pytest.raises(ValueError, match="need expects_clarification: true"): + TurnDefinition(turn_id="t1", message="m", expected_skill="search", clarifying_option_titles="visualization") + with pytest.raises(ValueError): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="search", + expects_clarification=True, + clarifying_option_titles="chart", + ) + + +_NSB_CANDIDATES = ["visualization/643699f3", "visualization/68ddf963"] + + +@pytest.mark.parametrize( + ("options", "off"), + [ + # gen-ai replies to "Describe the Net Sales Breakdown chart." on ecommerce_demo (gpt-5.6-luna). + (["{visualization/643699f3}", "{visualization/68ddf963}"], []), + (["Net Sales Breakdown", "Sales Goal"], ["Sales Goal"]), + (["Net Sales", "Sales Goal"], ["Net Sales", "Sales Goal"]), + (["Net Sales Breakdown (1. Overview)", "Net Sales Breakdown (2. Sales)"], []), + (["{visualization/8a07a527}", "{visualization/643699f3}"], ["visualization/8a07a527"]), + (["Donut chart", "Waterfall chart"], []), + (["The donut one", "Bullet chart"], ["Bullet chart"]), + (["Net Sales (donut)", "Waterfall chart"], ["Net Sales (donut)"]), + ], +) +def test_every_option_must_name_one_of_the_candidate_objects(options: list[str], off: list[str]) -> None: + card = { + "type": "clarifyingQuestions", + "questions": [{"text": "Which chart?", "control": {"options": [{"text": o} for o in options]}}], + } + turn_def = _clarifying_turn(expects_clarification=True, clarifying_option_objects=_NSB_CANDIDATES) + with patch("gooddata_eval.core.agentic.conversation.fetch_workspace_objects", return_value=_CHART_TITLES): + result, _ = _run([_card_reply(card), _result(viz=_RETURNS_VIZ)], [turn_def]) + expected = [f"options name none of {_NSB_CANDIDATES}: {off}"] if off else [] + assert result.turn_results[0].clarification_mismatches == expected + + +def test_a_chart_type_names_a_candidate_only_when_no_other_candidate_has_it() -> None: + reply = read_first_reply( + _result(unhandledParts=[_questionnaire(["Donut chart", "Waterfall chart"])]), asked_in_text=False + ) + same_type = _CHART_TITLES.model_copy(update={"chart_types": {_NSB_1: "donut", _NSB_2: "donut"}}) + assert off_candidate_options(reply, same_type, _NSB_CANDIDATES) == ["Donut chart", "Waterfall chart"] + + +def test_candidate_objects_accept_date_references() -> None: + turn = TurnDefinition( + turn_id="t1", + message="m", + expected_skill="visualization", + expects_clarification=True, + clarifying_option_objects=["date/order_date", "date_attribute/ship_date.month"], + ) + assert turn.clarifying_option_objects == ["date/order_date", "date_attribute/ship_date.month"] + + +@pytest.mark.parametrize("objects", [[], ["643699f3"], ["chart/643699f3"]]) +def test_candidate_objects_need_type_id_references(objects: list[str]) -> None: + with pytest.raises(ValueError, match="clarifying_option_objects needs type/id references"): + TurnDefinition( + turn_id="t1", + message="m", + expected_skill="search", + expects_clarification=True, + clarifying_option_objects=objects, + )