From 0ae3164c62b5a2f489488531363fa57cb87b68ee Mon Sep 17 00:00:00 2001 From: Roman Rakus Date: Tue, 3 Nov 2026 14:14:19 +0100 Subject: [PATCH] feat(gooddata-eval): gate a document item on asking before it drafts `document_asked_first` was recorded but never scored, and the simulated user always answered with the expected charts and period. An item could not require the copilot to ask, and a question about something else got an answer that did not fit. Two optional expectation keys close that: - `asked_first_required`: true makes `document_asked_first` a strict check. It fails a run that drafted without asking on an earlier turn. A run that never drafted fails only for the missing draft. The key needs `expects_clarification: true`. The diagnostic copy is then not published, so the score is not reported twice. - `clarification_reply`: the text the simulated user sends to the first turn that asks, in place of the reply built from the charts and period. Later questions get the built reply. The key needs `expects_clarification: true`. A turn counts as asking when `classify_reply` returns anything but `no_action`. A confirmation such as "Should I create the document anyway?" stops the copilot before it drafts, so it counts too. The diagnostic follows the same rule. `classify_reply` itself is unchanged, so metric and alert items keep their replies. A malformed value is rejected before the first request. Without either key, nothing changes for existing items. A refused `draft_document` call is not a draft, so a copilot that asks after the tool refuses an empty page counts as asking first. jira: LX-3243 risk: low Co-Authored-By: Claude Opus 5.5 --- .../core/agentic/document_skill.py | 58 +++++++++++--- .../tests/test_agentic_document_skill.py | 75 +++++++++++++++++++ 2 files changed, 122 insertions(+), 11 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index 533b0d40b..743c137d2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -6,7 +6,7 @@ import re import time from collections.abc import Iterator -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any from gooddata_eval.core.agentic._conversation_context import classify_reply @@ -65,7 +65,9 @@ # The copilot's own rule: a document opens with a cover and has at least one content page. _COVER_PAGE = "cover" _CONTENT_PAGE = "content" -_EXPECTATION_KEYS = frozenset({"period", "visualizations", "narrative", "expects_clarification"}) +_EXPECTATION_KEYS = frozenset( + {"period", "visualizations", "narrative", "expects_clarification", "asked_first_required", "clarification_reply"} +) # Template tokens the document renders at export time ({documentName}, {periodStart}, ...). _PLACEHOLDER_RE = re.compile(r"\{\w+\}") _SUMMARY_SLOT = "summary" @@ -299,6 +301,10 @@ def _expects_clarification(expected_output: dict) -> bool: return "expects_clarification" in expected_output +def _asked_first_required(expected_output: dict) -> bool: + return expected_output.get("asked_first_required") is True + + def _validate_expectation(expected_output: Any) -> None: """Reject a fixture the run could not score meaningfully, before the first API call. @@ -336,6 +342,19 @@ def _validate_expectation(expected_output: Any) -> None: raise ValueError( f"expects_clarification must be true or false, got {expected_output['expects_clarification']!r}" ) + if "asked_first_required" in expected_output: + if not isinstance(expected_output["asked_first_required"], bool): + raise ValueError( + f"asked_first_required must be true or false, got {expected_output['asked_first_required']!r}" + ) + if expected_output["asked_first_required"] and expected_output.get("expects_clarification") is not True: + raise ValueError("asked_first_required needs expects_clarification: true") + if "clarification_reply" in expected_output: + reply = expected_output["clarification_reply"] + if not isinstance(reply, str) or not reply.strip(): + raise ValueError(f"clarification_reply must be a non-empty reply, got {reply!r}") + if expected_output.get("expects_clarification") is not True: + raise ValueError("clarification_reply needs expects_clarification: true") def _validate_user_context(user_context: dict | None) -> None: @@ -351,12 +370,16 @@ def _validate_user_context(user_context: dict | None) -> None: raise ValueError("user_context.view.report is the Report name; send view.document") -def build_simulated_reply(expected_output: dict) -> str: +def build_simulated_reply(expected_output: dict, *, asked: bool = True) -> str: """The reply the simulated user sends when the copilot asks back instead of drafting. Deterministic and LLM-free, built from what the fixture states, so a failure stays the copilot's rather than a simulated user that phrased things differently each run. + A fixture's ``clarification_reply`` replaces it after a turn that ``asked``, for a question + the charts and period do not answer. The run sends it once, to the first such turn. """ + if asked and isinstance(reply := expected_output.get("clarification_reply"), str): + return reply segments: list[str] = [] visualizations = expected_output.get("visualizations") or [] if visualizations: @@ -380,6 +403,7 @@ class _Applies: period: bool charts: bool narrative: bool = False + asked_first: bool = False @dataclass @@ -398,6 +422,7 @@ class DocumentEvaluation: charts_matched: bool = False summaries_present: bool = False narrative_judged: bool = False + asked_first: bool = False judge_reasoning: str = "" # Set when the judge returned something unreadable. The run then has no narrative verdict: # it is neither published as a 0 nor allowed to pass on the remaining checks. @@ -438,6 +463,8 @@ def strict_checks(self) -> dict[str, bool]: checks["document_summaries_present"] = self.summaries_present if self.judge_error is None: checks["document_narrative_judged"] = self.narrative_judged + if self.applies.asked_first: + checks["document_asked_first"] = self.asked_first return checks @@ -646,10 +673,12 @@ def diagnostics(self) -> dict[str, bool]: """Observed but not scored. Whether the copilot asked before drafting is recorded only when the fixture says it - expects a question, and never gates: how much the copilot should ask is a product - decision, not something a run can get wrong. + expects a question. It gates only when the fixture sets ``asked_first_required``: how + much the copilot should ask is a product decision, made per item. """ - return {"document_asked_first": self.asked_first} if self.expects_clarification else {} + if not self.expects_clarification or self.evaluation.applies.asked_first: + return {} + return {"document_asked_first": self.asked_first} @property def judge_error(self) -> str | None: @@ -711,7 +740,8 @@ def _execute_single_document_run( turn_offset = 0.0 tool_index_offset = 0 reasoning_index_offset = 0 - first_turn_asked = False + asked_before_draft = False + replied = False run_started = time.monotonic() for iteration in range(max_iterations): @@ -748,8 +778,8 @@ def _execute_single_document_run( break response_text = (chat_result.text_response or "").strip() or render_answer_text(chat_result) - if iteration == 0: - first_turn_asked = classify_reply(chat_result, response_text) == "question" + asked = classify_reply(chat_result, response_text) != "no_action" + asked_before_draft = asked_before_draft or asked if not response_text and not chat_result.tool_call_events: break if iteration >= max_iterations - 1: @@ -758,7 +788,8 @@ def _execute_single_document_run( f"[timer] document_skill {conversation_id} GoodData turn {turns} complete after " f"{agent_elapsed:.2f}s; answering with the expected charts and period" ) - current_question = build_simulated_reply(expected_output) + current_question = build_simulated_reply(expected_output, asked=asked and not replied) + replied = replied or asked evaluation = evaluate_document_response( tool_result, @@ -767,6 +798,11 @@ def _execute_single_document_run( skill_activated=_skill_activated(all_tool_call_events, _BUILDER_SKILL), legacy_names=legacy_wire_names(all_parts, all_tool_call_events), ) + if _asked_first_required(expected_output): + evaluation.applies = replace(evaluation.applies, asked_first=True) + evaluation.asked_first = asked_before_draft + if tool_result is not None and not asked_before_draft: + evaluation.failures.append("the copilot drafted without asking first") if judge is not None and _has_narrative(expected_output): timings.judge_s += _judge_narrative(evaluation, judge, document_part, question, expected_output["narrative"]) @@ -774,7 +810,7 @@ def _execute_single_document_run( conversation_id=conversation_id, evaluation=evaluation, expects_clarification=_expects_clarification(expected_output), - asked_first=first_turn_asked, + asked_first=asked_before_draft, tool_result=tool_result, document_part=document_part, total_turns=turns, diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index 58c2d5f1f..55e131550 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -615,6 +615,10 @@ def test_the_document_format_is_recorded_in_the_run_metadata( ({"visualizations": []}, "visualizations is empty"), ({"visualizations": [{"title": "Revenue"}]}, "every visualization needs an 'id'"), ({"expects_clarification": "yes"}, "expects_clarification must be true or false"), + ({"asked_first_required": "yes"}, "asked_first_required must be true or false"), + ({"asked_first_required": True}, "asked_first_required needs expects_clarification: true"), + ({"clarification_reply": " "}, "clarification_reply must be a non-empty reply"), + ({"clarification_reply": "Use 2026."}, "clarification_reply needs expects_clarification: true"), ], ) def test_an_unusable_fixture_is_rejected_before_any_request(expected: dict, message: str) -> None: @@ -788,6 +792,77 @@ def test_a_structured_clarifying_question_counts_as_asking() -> None: assert run.diagnostics == {"document_asked_first": True} +def test_a_run_that_must_ask_first_fails_when_it_drafts_straight_away() -> None: + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([_drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is False + assert not run.evaluation.strict_pass + assert run.diagnostics == {} + + +def test_a_run_that_must_ask_first_passes_when_it_asks_then_drafts() -> None: + question = _chat_result(text="The pages show no data for 2030. Drop them, change the period, or keep them?") + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([question, _drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is True + assert run.evaluation.strict_pass + + +def test_a_run_that_must_ask_first_passes_when_it_asks_for_confirmation() -> None: + question = _chat_result(text="I could not find any data for 2030. Should I create the document anyway?") + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([question, _drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is True + assert run.evaluation.strict_pass + + +def test_a_run_that_must_ask_first_passes_when_it_asks_after_a_status_turn() -> None: + status = _chat_result(text="Working on it.") + question = _chat_result(text="Which period?") + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([status, question, _drafting_turn()]), "c", "q", expected, 4) + assert run.evaluation.strict_checks["document_asked_first"] is True + assert run.evaluation.strict_pass + + +def test_a_run_that_must_ask_first_and_never_drafts_fails_only_for_the_missing_draft() -> None: + expected = {"expects_clarification": True, "asked_first_required": True} + run = _execute_single_document_run(_ScriptedChatClient([_chat_result()]), "c", "q", expected, 4) + assert run.evaluation.failures == ["the agent never produced a successful draft_document call"] + + +def test_the_fixture_reply_answers_the_copilots_question() -> None: + client = _ScriptedChatClient([_chat_result(text="Which period?"), _drafting_turn()]) + expected = {"expects_clarification": True, "clarification_reply": "Use the first half of 2026 instead."} + _execute_single_document_run(client, "c", "q", expected, 4) + assert client.sent == ["q", "Use the first half of 2026 instead."] + + +def test_the_fixture_reply_answers_only_the_first_question() -> None: + client = _ScriptedChatClient( + [_chat_result(text="Which period?"), _chat_result(text="Which charts?"), _drafting_turn()] + ) + expected = {"expects_clarification": True, "clarification_reply": "Use the first half of 2026 instead."} + _execute_single_document_run(client, "c", "q", expected, 4) + assert client.sent == [ + "q", + "Use the first half of 2026 instead.", + "Anything else is up to you. Please create the document now.", + ] + + +def test_the_fixture_reply_waits_for_a_question() -> None: + expected = {"expects_clarification": True, "clarification_reply": "Use the first half of 2026 instead."} + status = _chat_result(text="Working on it.") + client = _ScriptedChatClient([status, _chat_result(text="Which period?"), _drafting_turn()]) + _execute_single_document_run(client, "c", "q", expected, 4) + assert client.sent == [ + "q", + "Anything else is up to you. Please create the document now.", + "Use the first half of 2026 instead.", + ] + + # ── page kinds, fixture keys, asking, two drafts in one turn ───────────────