diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index 533b0d40b..743c137d2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -6,7 +6,7 @@ import re import time from collections.abc import Iterator -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any from gooddata_eval.core.agentic._conversation_context import classify_reply @@ -65,7 +65,9 @@ # The copilot's own rule: a document opens with a cover and has at least one content page. _COVER_PAGE = "cover" _CONTENT_PAGE = "content" -_EXPECTATION_KEYS = frozenset({"period", "visualizations", "narrative", "expects_clarification"}) +_EXPECTATION_KEYS = frozenset( + {"period", "visualizations", "narrative", "expects_clarification", "asked_first_required", "clarification_reply"} +) # Template tokens the document renders at export time ({documentName}, {periodStart}, ...). _PLACEHOLDER_RE = re.compile(r"\{\w+\}") _SUMMARY_SLOT = "summary" @@ -299,6 +301,10 @@ def _expects_clarification(expected_output: dict) -> bool: return "expects_clarification" in expected_output +def _asked_first_required(expected_output: dict) -> bool: + return expected_output.get("asked_first_required") is True + + def _validate_expectation(expected_output: Any) -> None: """Reject a fixture the run could not score meaningfully, before the first API call. @@ -336,6 +342,19 @@ def _validate_expectation(expected_output: Any) -> None: raise ValueError( f"expects_clarification must be true or false, got {expected_output['expects_clarification']!r}" ) + if "asked_first_required" in expected_output: + if not isinstance(expected_output["asked_first_required"], bool): + raise ValueError( + f"asked_first_required must be true or false, got {expected_output['asked_first_required']!r}" + ) + if expected_output["asked_first_required"] and expected_output.get("expects_clarification") is not True: + raise ValueError("asked_first_required needs expects_clarification: true") + if "clarification_reply" in expected_output: + reply = expected_output["clarification_reply"] + if not isinstance(reply, str) or not reply.strip(): + raise ValueError(f"clarification_reply must be a non-empty reply, got {reply!r}") + if expected_output.get("expects_clarification") is not True: + raise ValueError("clarification_reply needs expects_clarification: true") def _validate_user_context(user_context: dict | None) -> None: @@ -351,12 +370,16 @@ def _validate_user_context(user_context: dict | None) -> None: raise ValueError("user_context.view.report is the Report name; send view.document") -def build_simulated_reply(expected_output: dict) -> str: +def build_simulated_reply(expected_output: dict, *, asked: bool = True) -> str: """The reply the simulated user sends when the copilot asks back instead of drafting. Deterministic and LLM-free, built from what the fixture states, so a failure stays the copilot's rather than a simulated user that phrased things differently each run. + A fixture's ``clarification_reply`` replaces it after a turn that ``asked``, for a question + the charts and period do not answer. The run sends it once, to the first such turn. """ + if asked and isinstance(reply := expected_output.get("clarification_reply"), str): + return reply segments: list[str] = [] visualizations = expected_output.get("visualizations") or [] if visualizations: @@ -380,6 +403,7 @@ class _Applies: period: bool charts: bool narrative: bool = False + asked_first: bool = False @dataclass @@ -398,6 +422,7 @@ class DocumentEvaluation: charts_matched: bool = False summaries_present: bool = False narrative_judged: bool = False + asked_first: bool = False judge_reasoning: str = "" # Set when the judge returned something unreadable. The run then has no narrative verdict: # it is neither published as a 0 nor allowed to pass on the remaining checks. @@ -438,6 +463,8 @@ def strict_checks(self) -> dict[str, bool]: checks["document_summaries_present"] = self.summaries_present if self.judge_error is None: checks["document_narrative_judged"] = self.narrative_judged + if self.applies.asked_first: + checks["document_asked_first"] = self.asked_first return checks @@ -646,10 +673,12 @@ def diagnostics(self) -> dict[str, bool]: """Observed but not scored. Whether the copilot asked before drafting is recorded only when the fixture says it - expects a question, and never gates: how much the copilot should ask is a product - decision, not something a run can get wrong. + expects a question. It gates only when the fixture sets ``asked_first_required``: how + much the copilot should ask is a product decision, made per item. """ - return {"document_asked_first": self.asked_first} if self.expects_clarification else {} + if not self.expects_clarification or self.evaluation.applies.asked_first: + return {} + return {"document_asked_first": self.asked_first} @property def judge_error(self) -> str | None: @@ -711,7 +740,8 @@ def _execute_single_document_run( turn_offset = 0.0 tool_index_offset = 0 reasoning_index_offset = 0 - first_turn_asked = False + asked_before_draft = False + replied = False run_started = time.monotonic() for iteration in range(max_iterations): @@ -748,8 +778,8 @@ def _execute_single_document_run( break response_text = (chat_result.text_response or "").strip() or render_answer_text(chat_result) - if iteration == 0: - first_turn_asked = classify_reply(chat_result, response_text) == "question" + asked = classify_reply(chat_result, response_text) != "no_action" + asked_before_draft = asked_before_draft or asked if not response_text and not chat_result.tool_call_events: break if iteration >= max_iterations - 1: @@ -758,7 +788,8 @@ def _execute_single_document_run( f"[timer] document_skill {conversation_id} GoodData turn {turns} complete after " f"{agent_elapsed:.2f}s; answering with the expected charts and period" ) - current_question = build_simulated_reply(expected_output) + current_question = build_simulated_reply(expected_output, asked=asked and not replied) + replied = replied or asked evaluation = evaluate_document_response( tool_result, @@ -767,6 +798,11 @@ def _execute_single_document_run( skill_activated=_skill_activated(all_tool_call_events, _BUILDER_SKILL), legacy_names=legacy_wire_names(all_parts, all_tool_call_events), ) + if _asked_first_required(expected_output): + evaluation.applies = replace(evaluation.applies, asked_first=True) + evaluation.asked_first = asked_before_draft + if tool_result is not None and not asked_before_draft: + evaluation.failures.append("the copilot drafted without asking first") if judge is not None and _has_narrative(expected_output): timings.judge_s += _judge_narrative(evaluation, judge, document_part, question, expected_output["narrative"]) @@ -774,7 +810,7 @@ def _execute_single_document_run( conversation_id=conversation_id, evaluation=evaluation, expects_clarification=_expects_clarification(expected_output), - asked_first=first_turn_asked, + asked_first=asked_before_draft, tool_result=tool_result, document_part=document_part, total_turns=turns, diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index 58c2d5f1f..55e131550 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -615,6 +615,10 @@ def test_the_document_format_is_recorded_in_the_run_metadata( ({"visualizations": []}, "visualizations is empty"), ({"visualizations": [{"title": "Revenue"}]}, "every visualization needs an 'id'"), ({"expects_clarification": "yes"}, "expects_clarification must be true or false"), + ({"asked_first_required": "yes"}, "asked_first_required must be true or false"), + ({"asked_first_required": True}, "asked_first_required needs expects_clarification: true"), + ({"clarification_reply": " "}, "clarification_reply must be a non-empty reply"), + ({"clarification_reply": "Use 2026."}, "clarification_reply needs expects_clarification: true"), ], ) def test_an_unusable_fixture_is_rejected_before_any_request(expected: dict, message: str) -> None: @@ -788,6 +792,77 @@ def test_a_structured_clarifying_question_counts_as_asking() -> None: assert run.diagnostics == {"document_asked_first": True} +def test_a_run_that_must_ask_first_fails_when_it_drafts_straight_away() -> None: + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([_drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is False + assert not run.evaluation.strict_pass + assert run.diagnostics == {} + + +def test_a_run_that_must_ask_first_passes_when_it_asks_then_drafts() -> None: + question = _chat_result(text="The pages show no data for 2030. Drop them, change the period, or keep them?") + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([question, _drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is True + assert run.evaluation.strict_pass + + +def test_a_run_that_must_ask_first_passes_when_it_asks_for_confirmation() -> None: + question = _chat_result(text="I could not find any data for 2030. Should I create the document anyway?") + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([question, _drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is True + assert run.evaluation.strict_pass + + +def test_a_run_that_must_ask_first_passes_when_it_asks_after_a_status_turn() -> None: + status = _chat_result(text="Working on it.") + question = _chat_result(text="Which period?") + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([status, question, _drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is True + assert run.evaluation.strict_pass + + +def test_a_run_that_must_ask_first_and_never_drafts_fails_only_for_the_missing_draft() -> None: + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([_chat_result()]), "c", "q", expected, 4) + assert run.evaluation.failures == ["the agent never produced a successful draft_document call"] + + +def test_the_fixture_reply_answers_the_copilots_question() -> None: + client = _ScriptedChatClient([_chat_result(text="Which period?"), _drafting_turn()]) + expected = {"expects_clarification": True, "clarification_reply": "Use the first half of 2026 instead."} + _execute_single_document_run(client, "c", "q", expected, 4) + assert client.sent == ["q", "Use the first half of 2026 instead."] + + +def test_the_fixture_reply_answers_only_the_first_question() -> None: + client = _ScriptedChatClient( + [_chat_result(text="Which period?"), _chat_result(text="Which charts?"), _drafting_turn()] + ) + expected = {"expects_clarification": True, "clarification_reply": "Use the first half of 2026 instead."} + _execute_single_document_run(client, "c", "q", expected, 4) + assert client.sent == [ + "q", + "Use the first half of 2026 instead.", + "Anything else is up to you. Please create the document now.", + ] + + +def test_the_fixture_reply_waits_for_a_question() -> None: + expected = {"expects_clarification": True, "clarification_reply": "Use the first half of 2026 instead."} + status = _chat_result(text="Working on it.") + client = _ScriptedChatClient([status, _chat_result(text="Which period?"), _drafting_turn()]) + _execute_single_document_run(client, "c", "q", expected, 4) + assert client.sent == [ + "q", + "Anything else is up to you. Please create the document now.", + "Use the first half of 2026 instead.", + ] + + # ── page kinds, fixture keys, asking, two drafts in one turn ───────────────