From be2a1d052cdbf205497d6476841a22d8bb7c76d9 Mon Sep 17 00:00:00 2001 From: "my.nguyen" Date: Tue, 17 Nov 2026 15:00:00 +0700 Subject: [PATCH] feat(gooddata-eval): per-item dashboard date filter in viz scoring A user who asks for a new chart from an open dashboard without naming a period may get the dashboard's date filter on the chart, and the product accepts that as well as an unfiltered chart. The strict date comparison failed it, which made up most failures of the _ctx visualization datasets. The item now says so with expected_output.date_filter_from_context: true, or {"dataset": ""}. Each candidate without a date filter is then also scored once per date filter in user_context.view.dashboard.filters, with that filter added; the comparison itself is check_filters, unchanged. The dataset comes from the key, else the dashboard filter's `using` (a bare id gets the dataset/ prefix), else, for the common date filter, the chart's only date filter when it is on a dataset/ URI. The key's dataset must be a bare id or a dataset/ URI, so a typo is a load error. A candidate stating a date filter is not extended. A per-item key rather than a run-wide flag: the verdict stays a function of the dataset item, an item that must reject the dashboard's period leaves the key out, and the Tavern shim gets it through the expectation it already relays, with no environment to set. A derived candidate replaces the best stated one only when it ranks higher, or ties with it and matches the date it misses, so a chart failing elsewhere is not also blamed for the date. Among stated candidates that tie, one that matched the date is preferred, so it keeps winning over a derived one. The candidate reported is always the stated one; the detail records filter_date_context and the failure summary marks the date. Items without the key score exactly as before. The single-shot runner now calls an optional Evaluator.validate before asking the agent, and the visualization evaluator checks its expectation there. A malformed item, or the key on an item whose context has no dashboard date filter, errors before any request on every path instead of after a spent chat turn. jira: QA-30214 risk: low Co-Authored-By: Claude Opus 5.5 --- packages/gooddata-eval/README.md | 41 ++ .../core/agentic/visualization.py | 34 +- .../core/evaluators/visualization.py | 200 ++++++- .../src/gooddata_eval/core/runner.py | 5 + .../tests/test_context_date_filter.py | 547 ++++++++++++++++++ 5 files changed, 820 insertions(+), 7 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_context_date_filter.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfda520f4..158c31187 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -752,6 +752,9 @@ Item-level keys, all optional except the candidates: `{"metric/net_sales": {"divide_by": 1000000, "decimals": 1, "suffix": "M"}}`. - `requires_visualization_part` — the chart must come back as a part of the answer, not only as the arguments of the chart tool call. +- `date_filter_from_context` — `true` or `{"dataset": ""}`: a candidate without a + date filter also accepts the chart carrying a date filter of the dashboard in the item's + `user_context`. See *Date filter carried over from the dashboard* below. Switches on a candidate chart: @@ -902,6 +905,44 @@ In a Langfuse dataset, put it in the item `metadata` (or in the `input` object) `null` value in `input` falls back to `metadata`. The value used must be an object or `null`; anything else fails the dataset load. +#### Date filter carried over from the dashboard + +A user who asks for a new chart from an open dashboard without naming a period may get the +dashboard's date filter on the chart, and the product accepts that as well as a chart with no +date filter. An item says so with `date_filter_from_context`: + +```json +"expected_output": { + "visualization": {"type": "headline_chart", "query": {"fields": {"m": {"using": "metric/gross_margin_pct"}}}, "metrics": ["m"]}, + "date_filter_from_context": {"dataset": "dataset/transaction_date"} +} +``` + +Each candidate without a date filter then also counts once more per date filter in +`user_context.view.dashboard.filters`, with that filter added. Both sides resolve against the +same day, so a relative and an absolute period that cover the same days match. The date dataset +of the added filter is, in order: the `dataset` the key states; the one the dashboard filter +names in `using` (a bare id such as `order_date` reads as `dataset/order_date`); for the +dashboard's common date filter, which names none, the one the chart's only date filter uses. +`true` therefore accepts any date dataset for the common date filter. The key's `dataset` must +be a bare id or a `dataset/` URI; anything else, such as `label/region` or a misspelled +`datset/order_date`, is an error. A dashboard filter whose `using` is not a date dataset adds +no candidate. + +A candidate that states a date filter is not extended: a question that names a period is scored +on that period only. A chart with two date filters matches no added candidate. Ranking and +attribute filters are unaffected. + +When the chart matches an added candidate, the run detail carries +`filter_date_context: {"inherited": true, "dataset": ...}` and a failure summary shows +`date : True (matches the dashboard date filter in user_context)`. The expected output reported +is still the candidate the item states; `expected_filters` shows the dashboard filter it was +scored with. A stated candidate that matched the date wins over an added one it ties with. Without the key an item is +scored exactly as before, so an item that must reject the dashboard's period ("all time") +simply leaves it out. The key on an item whose `user_context` has no dashboard date filter is +an error, raised before any request. Applies to `visualization`, `vis_agentic` and +`agentic_visualization` items; `agentic_conversation` turns do not read it. + `agentic_conversation` items take the item's `user_context` as the context of the first turn. A turn in `expected_output.turns` can set its own `user_context`, which applies from that turn on until another turn changes it, as the attached context does in the UI: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 87119eb89..b6ca2359a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -47,6 +47,7 @@ ExpectationOptions, _check_visualization_skill_activated, _evaluate_against_candidates, + check_context_date_filter, evaluation_result_detail, with_execution, with_options, @@ -213,6 +214,7 @@ def _execute_single_run( max_iterations: int = _DEFAULT_MAX_ITERATIONS, requires_execution: bool = False, options: ExpectationOptions | None = None, + user_context: dict | None = None, ) -> RunResult: """Drive one full multi-turn conversation and evaluate the result.""" options = _effective_options(options, requires_execution) @@ -314,7 +316,13 @@ def _execute_single_run( if current_result.created_visualizations and current_result.created_visualizations.objects: actual_output = current_result.created_visualizations.objects[0] - eval_result, best_expected = _evaluate_against_candidates(expected_outputs, actual_output, skill_activated) + eval_result, best_expected = _evaluate_against_candidates( + expected_outputs, + actual_output, + skill_activated, + user_context=user_context, + context_date_filter=options.date_filter_from_context, + ) eval_result = with_execution( eval_result, all_tool_call_events, current_result.text_response, options.requires_execution ) @@ -379,6 +387,8 @@ def run_agentic_visualization( ValueError: ``expected_outputs`` and ``options.expects_visualization`` disagree. An empty candidate list is only right for an item that expects no chart, and the scorer would otherwise read the default ``expects_visualization`` and fail it. + Also raised, before any request, for ``date_filter_from_context`` on an item whose + ``user_context`` has no dashboard date filter (``check_context_date_filter``). """ expects_visualization = (options or ExpectationOptions()).expects_visualization if expects_visualization != bool(expected_outputs): @@ -386,6 +396,7 @@ def run_agentic_visualization( "expected_outputs must list the candidate charts unless options.expects_visualization is false, " f"got {len(expected_outputs)} candidate(s) with expects_visualization={expects_visualization}" ) + check_context_date_filter(options or ExpectationOptions(), user_context) client = ChatClient( host=host, token=token, @@ -401,7 +412,14 @@ def run_agentic_visualization( try: run_results.append( _execute_single_run( - client, conv_id_0, question, expected_outputs, max_iterations, requires_execution, options + client, + conv_id_0, + question, + expected_outputs, + max_iterations, + requires_execution, + options, + user_context=user_context, ) ) finally: @@ -413,7 +431,14 @@ def run_agentic_visualization( try: run_results.append( _execute_single_run( - client, conv_id, question, expected_outputs, max_iterations, requires_execution, options + client, + conv_id, + question, + expected_outputs, + max_iterations, + requires_execution, + options, + user_context=user_context, ) ) finally: @@ -789,7 +814,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f" expected : {sorted(ev.expected_dim_uris)}\n" f" actual : {sorted(ev.actual_dim_uris)}\n" f" Filters Correct : {ev.filters_correct}\n" - f" date : {ev.filter_date_score}\n" + f" date : {ev.filter_date_score}" + f"{' (matches the dashboard date filter in user_context)' if ev.filter_date_inherited else ''}\n" f"{_filter_diff('date', ev)}" f" ranking : {ev.filter_ranking_score}\n" f"{_filter_diff('ranking', ev)}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py index 10806d047..a0265ad12 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py @@ -83,6 +83,15 @@ class EvaluationResult: format_correct: bool | None = None format_problems: list[str] = field(default_factory=list) part_present: bool | None = None + # Set only when the best candidate is one `date_filter_from_context` derived from the + # dashboard in the item's `user_context` and its date filter matched: the date dataset + # that filter used. Reported in the detail only when set. + filter_date_inherited_dataset: str | None = None + + @property + def filter_date_inherited(self) -> bool: + """Whether the date filter matched one taken from the dashboard in `user_context`.""" + return self.filter_date_inherited_dataset is not None @property def scored_without_chart(self) -> bool: @@ -248,6 +257,19 @@ def requires_execution_of(expected_output: object) -> bool: return isinstance(expected_output, dict) and expected_output.get("requires_execution") is True +@dataclass(frozen=True) +class ContextDateFilter: + """The item-level ``date_filter_from_context`` key. + + Every candidate without a date filter also accepts the chart carrying one of the date + filters of the dashboard in the item's ``user_context``. ``dataset`` is the date dataset + that filter must use; None takes the one the dashboard filter names, and for the + dashboard's common date filter, which names none, the one the chart used. + """ + + dataset: str | None = None + + @dataclass(frozen=True) class ExpectationOptions: """The item-level keys of a visualization expectation, beside its candidate charts. @@ -266,6 +288,10 @@ class ExpectationOptions: - ``format_intents``: per metric uri, how its numbers must read (see ``number_format``). - ``requires_visualization_part``: the chart must come back as a part of the answer. Without it, a chart that exists only in the tool arguments is scored like one the user saw. + - ``date_filter_from_context``: the chart may carry the date filter of the dashboard in the + item's ``user_context`` instead of none (see ``ContextDateFilter``). For a question that + names no period asked from an open dashboard, where carrying the dashboard's period over + and leaving the chart unfiltered are both correct. """ requires_execution: bool = False @@ -275,6 +301,69 @@ class ExpectationOptions: answer_must_not_match: tuple[str, ...] = () format_intents: Mapping[str, Mapping[str, Any] | None] | None = None requires_visualization_part: bool = False + date_filter_from_context: ContextDateFilter | None = None + + +_DATASET_PREFIX = "dataset/" + + +def date_dataset_uri(dataset: object) -> str | None: + """``dataset`` as a ``dataset/`` URI; a bare id, as dashboard filters name it, gets the prefix. + + None for anything else: not a non-empty string, or a URI of another type such as + ``label/region`` or a misspelled ``datset/order_date``. + """ + if not isinstance(dataset, str) or not dataset: + return None + if dataset.startswith(_DATASET_PREFIX): + return dataset if len(dataset) > len(_DATASET_PREFIX) else None + return None if "/" in dataset else f"{_DATASET_PREFIX}{dataset}" + + +def _context_date_filter(expected_output: dict) -> ContextDateFilter | None: + raw = expected_output.get("date_filter_from_context", False) + if raw is False: + return None + if raw is True: + return ContextDateFilter() + if isinstance(raw, dict) and set(raw) == {"dataset"}: + dataset = date_dataset_uri(raw["dataset"]) + if dataset is not None: + return ContextDateFilter(dataset=dataset) + raise ValueError( + 'date_filter_from_context must be true, false or {"dataset": ""} with a bare date dataset id ' + f"or a dataset/ URI, got {raw!r}" + ) + + +def dashboard_date_filters(user_context: object) -> list[dict]: + """The date filters of the dashboard open in ``user_context`` (``view.dashboard.filters``). + + Anything not shaped like that -- no context, no dashboard, a non-list ``filters``, an + entry that is not a ``date_filter`` object -- contributes nothing rather than raising: + the context is the chat API's schema and the dataset loader does not validate it. + """ + if not isinstance(user_context, Mapping): + return [] + view = user_context.get("view") + dashboard = view.get("dashboard") if isinstance(view, Mapping) else None + filters = dashboard.get("filters") if isinstance(dashboard, Mapping) else None + if not isinstance(filters, list): + return [] + return [f for f in filters if isinstance(f, dict) and f.get("type") == "date_filter"] + + +def check_context_date_filter(options: ExpectationOptions, user_context: object) -> None: + """Refuse ``date_filter_from_context`` on an item whose context has no dashboard date filter. + + Raises: + ValueError: the key is set and ``user_context`` offers nothing to take a date filter + from, so the key would silently accept nothing. + """ + if options.date_filter_from_context is not None and not dashboard_date_filters(user_context): + raise ValueError( + "date_filter_from_context is set, but the item's user_context has no date filter in view.dashboard.filters" + ) def _regex_error(pattern: str) -> str | None: @@ -308,6 +397,7 @@ def _patterns(expected_output: dict, key: str) -> tuple[str, ...]: "answer_must_not_match", "format_intents", "requires_visualization_part", + "date_filter_from_context", } ) @@ -367,6 +457,9 @@ def expectation_options(expected_output: object) -> ExpectationOptions: validate_format_intents(intents) if not expects and (intents is not None or part): raise ValueError("an expectation without a chart cannot state format_intents or requires_visualization_part") + context_date = _context_date_filter(expected_output) + if not expects and context_date is not None: + raise ValueError("an expectation without a chart cannot state date_filter_from_context") return ExpectationOptions( requires_execution=requires_execution_of(expected_output), expects_visualization=expects, @@ -375,6 +468,7 @@ def expectation_options(expected_output: object) -> ExpectationOptions: answer_must_not_match=_patterns(expected_output, "answer_must_not_match"), format_intents=intents, requires_visualization_part=part, + date_filter_from_context=context_date, ) @@ -551,17 +645,93 @@ def _evaluate_against_candidates( actual: CreatedVisualization | None, skill_activated: bool = False, today: date | None = None, + *, + user_context: object = None, + context_date_filter: ContextDateFilter | None = None, ) -> tuple[EvaluationResult, CreatedVisualization | None]: + """Score ``actual`` against every candidate and return the best result with its candidate. + + The returned candidate is always one the item states. With ``context_date_filter``, the + candidates ``context_date_candidates`` derives are scored after the stated ones and can + replace the best stated result only when they rank higher, or when they tie with it and + match the date it misses. Among stated candidates that tie, one that matched the date is + that best result -- so a chart failing on something else is not also blamed for + the dashboard's period, and a stated candidate that matched the date keeps winning. The + result is then marked with the date dataset, and the candidate returned is the stated one + the winner was derived from. + """ if not expected_outputs: return _no_candidate_result(actual, skill_activated), None # Anchored once here too: the candidates are ranked against each other, so letting # each resolve its own today could rank them on spans from different days. today = today or date.today() - pairs = [(_evaluate_visualization(exp, actual, skill_activated, today), exp) for exp in expected_outputs] - best_result, best_expected = max(pairs, key=lambda p: (p[0].strict_pass, p[0].strict_checks_passed_count)) + stated = [(_evaluate_visualization(exp, actual, skill_activated, today), exp) for exp in expected_outputs] + if context_date_filter is None: + return max(stated, key=lambda pair: _rank(pair[0])) + # With the key, a stated candidate that matched the date wins the stated ones it ties + # with, so the derived candidates below are measured against it. + best_result, best_expected = max(stated, key=lambda pair: (*_rank(pair[0]), pair[0].filter_date_score)) + for derived, origin, dataset in context_date_candidates( + expected_outputs, actual, user_context, context_date_filter + ): + result = _evaluate_visualization(derived, actual, skill_activated, today) + if not result.filter_date_score: + continue + ranks_higher = _rank(result) > _rank(best_result) + fixes_the_date = _rank(result) == _rank(best_result) and not best_result.filter_date_score + if ranks_higher or fixes_the_date: + best_result = replace(result, filter_date_inherited_dataset=dataset) + best_expected = origin return best_result, best_expected +def _rank(result: EvaluationResult) -> tuple[bool, int]: + return result.strict_pass, result.strict_checks_passed_count + + +def _date_filters(viz: CreatedVisualization) -> list[dict]: + return [f for f in viz.query.filter_by.values() if f.get("type") == "date_filter"] + + +def context_date_candidates( + expected_outputs: list[CreatedVisualization], + actual: CreatedVisualization | None, + user_context: object, + context_date_filter: ContextDateFilter, +) -> list[tuple[CreatedVisualization, CreatedVisualization, str]]: + """Each candidate without a date filter, once per dashboard date filter in ``user_context``. + + Every copy carries that dashboard filter, on the date dataset ``ContextDateFilter`` picks, + and is returned with the candidate it was copied from and that dataset. A dashboard + filter naming no dataset, with no dataset stated on the item, takes the dataset of the + chart's only date filter; with no such filter, or one whose ``using`` is not a ``dataset/`` + URI, there is nothing for it to match, and it yields no copy. A dashboard filter whose + ``using`` is not a date dataset yields none either. + """ + actual_dates = _date_filters(actual) if actual is not None else [] + actual_using = actual_dates[0].get("using") if len(actual_dates) == 1 else None + actual_dataset = actual_using if date_dataset_uri(actual_using) == actual_using else None + derived: list[tuple[CreatedVisualization, CreatedVisualization, str]] = [] + for expected in expected_outputs: + if _date_filters(expected): + continue + for dashboard_filter in dashboard_date_filters(user_context): + dashboard_using = dashboard_filter.get("using") + dashboard_dataset = date_dataset_uri(dashboard_using) + if dashboard_using and dashboard_dataset is None: + continue + dataset = context_date_filter.dataset or dashboard_dataset or actual_dataset + if dataset is None: + continue + key = "context_date_filter" + while key in expected.query.filter_by: + key = f"_{key}" + filter_by = {**expected.query.filter_by, key: {**dashboard_filter, "using": dataset}} + query = expected.query.model_copy(update={"filter_by": filter_by}) + derived.append((expected.model_copy(update={"query": query}), expected, dataset)) + return derived + + def _parse_expected(expected_output: dict) -> list[CreatedVisualization]: if not isinstance(expected_output, dict): raise ValueError("'expected_output' must be a JSON object") @@ -624,6 +794,13 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict: "actual_dim_uris": sorted(ev.actual_dim_uris), "expected_filters": ev.expected_filters, "actual_filters": ev.actual_filters, + # Nested and only when it applies: `quality_score` counts every top-level boolean, and + # an item without `date_filter_from_context` must not carry the key at all. + **( + {"filter_date_context": {"inherited": True, "dataset": ev.filter_date_inherited_dataset}} + if ev.filter_date_inherited + else {} + ), "expected_sorts": ev.expected_sorts, "actual_sorts": ev.actual_sorts, **({"metric_layout_problems": ev.metric_layout_problems} if ev.metric_layout_problems else {}), @@ -653,12 +830,29 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict: class VisualizationEvaluator: test_kind = "visualization" + def validate(self, item: DatasetItem) -> None: + """Refuse a malformed expectation before the runner sends the item's question. + + Raises: + ValueError: an item-level key or a candidate chart is malformed, or + ``date_filter_from_context`` has no dashboard date filter to take a filter from. + """ + check_context_date_filter(expectation_options(item.expected_output), item.user_context) + _parse_expected(item.expected_output) + def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation: options = expectation_options(item.expected_output) + check_context_date_filter(options, item.user_context) candidates = _parse_expected(item.expected_output) actual = _extract_actual(chat_result) skill_activated = _check_visualization_skill_activated(chat_result.tool_call_events) - ev, _best_expected = _evaluate_against_candidates(candidates, actual, skill_activated) + ev, _best_expected = _evaluate_against_candidates( + candidates, + actual, + skill_activated, + user_context=item.user_context, + context_date_filter=options.date_filter_from_context, + ) ev = with_execution(ev, chat_result.tool_call_events, chat_result.text_response, options.requires_execution) ev = with_options(ev, options, actual, chat_result.tool_call_events, render_answer_text(chat_result)) return ItemEvaluation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index c93488faa..b0c8d0a43 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -295,7 +295,12 @@ def _run_one_item( best_chat_result: ChatResult | None = None best_run_latency: float | None = None judge_errors: list[str] = [] + # Optional on the Evaluator protocol: an evaluator that can tell a malformed item from its + # expectation alone refuses it here, before the item's question is sent to the agent. + validate = getattr(evaluator, "validate", None) try: + if validate is not None: + validate(item) for run_index in range(1, runs + 1): t0 = time.perf_counter() chat_result = backend.ask(item) diff --git a/packages/gooddata-eval/tests/test_context_date_filter.py b/packages/gooddata-eval/tests/test_context_date_filter.py new file mode 100644 index 000000000..080603348 --- /dev/null +++ b/packages/gooddata-eval/tests/test_context_date_filter.py @@ -0,0 +1,547 @@ +# (C) 2026 GoodData Corporation +"""The item-level `date_filter_from_context` key: a date filter taken from the dashboard in `user_context`. + +Without the key, every case here must score exactly as the stated candidates alone do. +""" + +import json +from datetime import date +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic.visualization import ( + AgenticRunSummary, + VisualizationAssertionError, + evaluate_agentic_visualization, + run_agentic_visualization, +) +from gooddata_eval.core.evaluators import get_evaluator +from gooddata_eval.core.evaluators.visualization import ( + ContextDateFilter, + EvaluationResult, + ExpectationOptions, + _evaluate_against_candidates, + check_context_date_filter, + context_date_candidates, + dashboard_date_filters, + date_dataset_uri, + evaluation_result_detail, + expectation_options, +) +from gooddata_eval.core.models import ChatResult, CreatedVisualization, DatasetItem +from gooddata_eval.core.runner import _run_one_item + +_TODAY = date(2026, 11, 12) + +# The dashboard of globalmart_visualization_charts_ctx: a common date filter, no dataset named. +_USER_CONTEXT = { + "view": { + "dashboard": { + "id": "dashboard_033", + "title": "Order Lines & Margin", + "filters": [{"type": "date_filter", "granularity": "MONTH", "from": -11, "to": 0}], + } + } +} +_INHERITED = { + "type": "date_filter", + "using": "dataset/transaction_date", + "granularity": "MONTH", + "from": -11, + "to": 0, +} +_ANY = ContextDateFilter() +_TRANSACTION_DATE = ContextDateFilter(dataset="dataset/transaction_date") + + +def _headline(filter_by: dict | None = None, metric: str = "metric/gross_margin_percent") -> dict: + return { + "id": "v", + "type": "headline_chart", + "query": {"fields": {"m": {"using": metric}}, "filter_by": filter_by or {}}, + "metrics": ["m"], + } + + +def _viz(filter_by: dict | None = None) -> CreatedVisualization: + return CreatedVisualization.model_validate(_headline(filter_by)) + + +def _date(frm: int, to: int, granularity: str = "MONTH", using: str = "dataset/transaction_date") -> dict: + return {"type": "date_filter", "using": using, "granularity": granularity, "from": frm, "to": to} + + +def _context(*filters: dict) -> dict: + return {"view": {"dashboard": {"id": "d", "filters": list(filters)}}} + + +def _score( + actual_filter_by: dict | None, + expected_filter_by: dict | None = None, + key: ContextDateFilter | None = _ANY, + user_context: object = _USER_CONTEXT, +) -> EvaluationResult: + ev, _ = _evaluate_against_candidates( + [_viz(expected_filter_by)], + _viz(actual_filter_by), + True, + _TODAY, + user_context=user_context, + context_date_filter=key, + ) + return ev + + +# ── parsing the key ────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("raw", "parsed"), + [ + (False, None), + (True, _ANY), + ({"dataset": "dataset/transaction_date"}, _TRANSACTION_DATE), + ({"dataset": "transaction_date"}, _TRANSACTION_DATE), + ], +) +def test_the_key_parses(raw: object, parsed: ContextDateFilter | None) -> None: + options = expectation_options({"visualization": _headline(), "date_filter_from_context": raw}) + assert options.date_filter_from_context == parsed + + +def test_an_absent_key_parses_to_none() -> None: + assert expectation_options({"visualization": _headline()}).date_filter_from_context is None + + +@pytest.mark.parametrize( + "raw", + [ + None, + "yes", + 1, + {}, + {"dataset": ""}, + {"dataset": 1}, + {"dataset": "x", "extra": 1}, + {"dataset": "datset/transaction_date"}, + {"dataset": "label/region"}, + {"dataset": "dataset/"}, + ], +) +def test_a_malformed_key_is_refused(raw: object) -> None: + with pytest.raises(ValueError, match="date_filter_from_context"): + expectation_options({"visualization": _headline(), "date_filter_from_context": raw}) + + +def test_the_key_on_an_item_without_a_chart_is_refused() -> None: + with pytest.raises(ValueError, match="date_filter_from_context"): + expectation_options({"expects_visualization": False, "date_filter_from_context": True}) + + +@pytest.mark.parametrize("user_context", [None, {}, {"view": {}}, {"view": {"dashboard": {"id": "d"}}}, "text"]) +def test_the_key_without_a_dashboard_date_filter_in_context_is_refused(user_context: object) -> None: + with pytest.raises(ValueError, match="no date filter"): + check_context_date_filter(ExpectationOptions(date_filter_from_context=_ANY), user_context) + + +def test_without_the_key_any_context_is_fine() -> None: + check_context_date_filter(ExpectationOptions(), None) + + +@pytest.mark.parametrize( + ("raw", "uri"), + [ + ("order_date", "dataset/order_date"), + ("dataset/order_date", "dataset/order_date"), + ("datset/order_date", None), + ("label/region", None), + ("dataset/", None), + ("", None), + (None, None), + ], +) +def test_date_dataset_uri(raw: object, uri: str | None) -> None: + assert date_dataset_uri(raw) == uri + + +@pytest.mark.parametrize("malformed", ["text", {"view": "x"}, {"view": {"dashboard": {"filters": "x"}}}]) +def test_a_malformed_context_yields_no_dashboard_filter(malformed: object) -> None: + assert dashboard_date_filters(malformed) == [] + + +# ── scoring with the key ───────────────────────────────────────────────────── + + +def test_the_dashboard_period_passes_and_is_marked_inherited() -> None: + ev = _score({"f_transaction_date": _INHERITED}) + assert ev.strict_pass is True + assert ev.filter_date_score is True + assert ev.filter_date_inherited_dataset == "dataset/transaction_date" + + +def test_no_date_filter_still_passes_without_being_inherited() -> None: + ev = _score({}) + assert ev.strict_pass is True + assert ev.filter_date_inherited is False + + +def test_a_period_other_than_the_dashboards_fails() -> None: + ev = _score({"f": _date(-5, 0)}) + assert ev.filter_date_score is False + assert ev.filter_date_inherited is False + + +@pytest.mark.parametrize( + ("expected_filter", "actual_filter", "date_ok"), + [ + (_date(-1, -1, "YEAR"), _INHERITED, False), + (_date(-1, -1, "YEAR"), _date(-1, -1, "YEAR"), True), + (_INHERITED, _INHERITED, True), + ], +) +def test_a_candidate_stating_a_date_filter_is_not_extended( + expected_filter: dict, actual_filter: dict, date_ok: bool +) -> None: + ev = _score({"f": actual_filter}, {"f": expected_filter}) + assert ev.filter_date_score is date_ok + assert ev.filter_date_inherited is False + + +def test_the_common_filter_with_true_takes_the_charts_dataset() -> None: + ev = _score({"f": _date(-11, 0, using="dataset/order_date")}) + assert ev.filter_date_score is True + assert ev.filter_date_inherited_dataset == "dataset/order_date" + + +@pytest.mark.parametrize( + ("actual_dataset", "date_ok"), [("dataset/transaction_date", True), ("dataset/ship_date", False)] +) +def test_a_stated_dataset_accepts_only_that_one(actual_dataset: str, date_ok: bool) -> None: + ev = _score({"f": _date(-11, 0, using=actual_dataset)}, key=_TRANSACTION_DATE) + assert ev.filter_date_score is date_ok + assert ev.filter_date_inherited is date_ok + + +@pytest.mark.parametrize("dashboard_using", ["order_date", "dataset/order_date"]) +@pytest.mark.parametrize(("actual_dataset", "date_ok"), [("dataset/order_date", True), ("dataset/ship_date", False)]) +def test_a_dashboard_filter_naming_a_dataset_accepts_only_that_one( + dashboard_using: str, actual_dataset: str, date_ok: bool +) -> None: + context = _context({"type": "date_filter", "using": dashboard_using, "granularity": "MONTH", "from": -11, "to": 0}) + ev = _score({"f": _date(-11, 0, using=actual_dataset)}, user_context=context) + assert ev.filter_date_score is date_ok + + +def test_a_stated_dataset_wins_over_the_dashboard_filters() -> None: + context = _context(_date(-11, 0, using="dataset/order_date")) + ev = _score({"f": _INHERITED}, key=_TRANSACTION_DATE, user_context=context) + assert ev.filter_date_inherited_dataset == "dataset/transaction_date" + + +def test_a_chart_date_filter_naming_no_dataset_matches_nothing() -> None: + nameless = {k: v for k, v in _date(-11, 0).items() if k != "using"} + ev = _score({"f": nameless}) + assert ev.filter_date_score is False + + +def test_the_same_span_written_absolutely_is_accepted() -> None: + absolute = {"type": "date_filter", "using": "dataset/transaction_date", "from": "2025-12-01", "to": "2026-11-30"} + ev = _score({"f": absolute}) + assert ev.filter_date_score is True + assert ev.filter_date_inherited is True + + +def test_two_chart_date_filters_fail_even_if_one_is_the_dashboards() -> None: + ev = _score({"a": _INHERITED, "b": _date(-1, -1, "YEAR", using="dataset/order_date")}) + assert ev.filter_date_score is False + assert ev.filter_date_inherited is False + + +def test_any_of_several_dashboard_date_filters_matches() -> None: + context = _context( + {"type": "attribute_filter", "using": "label/region"}, + {"type": "date_filter", "granularity": "YEAR", "from": 0, "to": 0}, + {"type": "date_filter", "granularity": "MONTH", "from": -11, "to": 0}, + ) + assert len(dashboard_date_filters(context)) == 2 + assert _score({"f": _INHERITED}, user_context=context).filter_date_score is True + + +def test_ranking_and_attribute_filters_are_not_relaxed() -> None: + attribute = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}} + ev = _score({"f": _INHERITED, "r": attribute}) + assert ev.filter_date_score is True + assert ev.filter_attribute_score is False + assert ev.strict_pass is False + + +def test_derived_candidates_keep_the_stated_filters() -> None: + attribute = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}} + expected = _viz({"context_date_filter": attribute}) + derived = context_date_candidates([expected], _viz({"f": _INHERITED}), _USER_CONTEXT, _ANY) + assert len(derived) == 1 + candidate, origin, dataset = derived[0] + assert origin is expected + assert dataset == "dataset/transaction_date" + assert candidate.query.filter_by["context_date_filter"] == attribute + assert candidate.query.filter_by["_context_date_filter"] == _INHERITED + assert expected.query.filter_by == {"context_date_filter": attribute} + + +def test_a_dashboard_filter_whose_using_is_not_a_date_dataset_adds_no_candidate() -> None: + context = _context({"type": "date_filter", "using": "label/region", "granularity": "MONTH", "from": -11, "to": 0}) + assert context_date_candidates([_viz()], _viz({"f": _INHERITED}), context, _ANY) == [] + assert _score({"f": _INHERITED}, user_context=context).filter_date_score is False + + +def test_a_matched_derived_candidate_reports_the_stated_one() -> None: + expected = [_viz()] + ev, best = _evaluate_against_candidates( + expected, _viz({"f": _INHERITED}), True, _TODAY, user_context=_USER_CONTEXT, context_date_filter=_ANY + ) + assert best is expected[0] + assert ev.filter_date_inherited_dataset == "dataset/transaction_date" + assert len(ev.expected_filters["date"]) == 1 + + +def test_a_stated_candidate_matching_the_date_beats_a_derived_one_it_ties_with() -> None: + stated_dated, stated_plain = _viz({"f": _INHERITED}), _viz() + wrong_metric = CreatedVisualization.model_validate(_headline({"f": _INHERITED}, metric="metric/revenue")) + ev, best = _evaluate_against_candidates( + [stated_dated, stated_plain], + wrong_metric, + True, + _TODAY, + user_context=_USER_CONTEXT, + context_date_filter=_ANY, + ) + assert best is stated_dated + assert ev.filter_date_score is True + assert ev.filter_date_inherited is False + + +def test_a_stated_candidate_matching_the_date_wins_the_stated_ones_it_ties_with() -> None: + region = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}} + other_region = {**region, "state": {"include": ["APAC"]}} + right_metric_wrong_region = _viz({"a": other_region}) + wrong_metric_right_filters = CreatedVisualization.model_validate( + _headline({"d": _INHERITED, "a": region}, metric="metric/revenue") + ) + candidates = [right_metric_wrong_region, wrong_metric_right_filters] + actual = _viz({"d": _INHERITED, "a": region}) + ev, best = _evaluate_against_candidates( + candidates, actual, True, _TODAY, user_context=_USER_CONTEXT, context_date_filter=_ANY + ) + assert best is wrong_metric_right_filters + assert ev.filter_date_inherited is False + _, best_without_key = _evaluate_against_candidates(candidates, actual, True, _TODAY) + assert best_without_key is right_metric_wrong_region + + +@pytest.mark.parametrize("using", ["label/region", "transaction_date", "datset/transaction_date"]) +def test_a_chart_date_filter_not_on_a_dataset_uri_matches_nothing(using: str) -> None: + ev = _score({"f": _date(-11, 0, using=using)}) + assert ev.filter_date_score is False + assert ev.strict_pass is False + assert ev.filter_date_inherited is False + + +def test_the_stated_candidate_is_returned_when_it_ties() -> None: + expected = [_viz()] + _, best = _evaluate_against_candidates( + expected, _viz(), True, _TODAY, user_context=_USER_CONTEXT, context_date_filter=_ANY + ) + assert best is expected[0] + + +# ── without the key nothing changes ────────────────────────────────────────── + + +@pytest.mark.parametrize( + "actual_filter_by", + [{}, {"f": _INHERITED}, {"f": _date(-5, 0)}, {"a": _INHERITED, "b": _date(0, 0, "YEAR")}], +) +@pytest.mark.parametrize("expected_filter_by", [{}, {"f": _date(-1, -1, "YEAR")}]) +def test_without_the_key_the_detail_is_byte_identical(expected_filter_by: dict, actual_filter_by: dict) -> None: + expected, actual = [_viz(expected_filter_by)], _viz(actual_filter_by) + before, _ = _evaluate_against_candidates(expected, actual, True, _TODAY) + after, _ = _evaluate_against_candidates(expected, actual, True, _TODAY, user_context=_USER_CONTEXT) + assert after == before + dump = json.dumps(evaluation_result_detail(before), sort_keys=True, default=sorted) + assert json.dumps(evaluation_result_detail(after), sort_keys=True, default=sorted) == dump + assert "filter_date_context" not in dump + + +# ── the item from the ticket, through both evaluation paths ───────────────── + + +def _item(expected_output: dict) -> DatasetItem: + return DatasetItem( + id="globalmart_visualization_charts_ctx_2", + dataset_name="globalmart_visualization_charts_ctx", + test_kind="visualization", + question="Show Gross Margin Percent as a headline.", + expected_output=expected_output, + user_context=_USER_CONTEXT, + ) + + +def _chat(metric: str = "metric/gross_margin_percent", filter_by: dict | None = None) -> ChatResult: + return ChatResult.model_validate( + { + "createdVisualizations": { + "objects": [_headline(filter_by or {"f_transaction_date": _INHERITED}, metric=metric)], + "reasoning": "", + }, + "toolCallEvents": [], + } + ) + + +def test_single_shot_fails_without_the_key() -> None: + result = get_evaluator("visualization").evaluate(_item({"visualization": _headline()}), _chat()) + assert result.passed is False + assert result.detail["filter_date_score"] is False + assert "filter_date_context" not in result.detail + + +def test_single_shot_passes_with_the_key() -> None: + item = _item({"visualization": _headline(), "date_filter_from_context": True}) + result = get_evaluator("visualization").evaluate(item, _chat()) + assert result.passed is True + assert result.detail["filter_date_score"] is True + assert result.detail["filter_date_context"] == {"inherited": True, "dataset": "dataset/transaction_date"} + + +def test_single_shot_refuses_the_key_without_a_dashboard_date_filter() -> None: + item = _item({"visualization": _headline(), "date_filter_from_context": True}) + item = item.model_copy(update={"user_context": None}) + with pytest.raises(ValueError, match="no date filter"): + get_evaluator("visualization").evaluate(item, _chat()) + + +@pytest.mark.parametrize( + ("expected_output", "user_context", "error"), + [ + ({"visualization": _headline(), "date_filter_from_context": True}, None, "no date filter"), + ({"visualization": _headline(), "date_filter_from_context": {"dataset": "label/x"}}, _USER_CONTEXT, "dataset"), + ({"visualization": _headline(), "answer_must_inlcude": ["x"]}, _USER_CONTEXT, "answer_must_inlcude"), + ], +) +def test_single_shot_runner_refuses_a_malformed_item_before_asking( + expected_output: dict, user_context: dict | None, error: str +) -> None: + item = _item(expected_output).model_copy(update={"user_context": user_context}) + backend = MagicMock() + report = _run_one_item(item, backend, runs=2) + backend.ask.assert_not_called() + assert report.runs == 0 + assert report.error is not None + assert report.error.startswith("ValueError:") + assert error in report.error + + +def test_single_shot_runner_asks_for_a_valid_item() -> None: + item = _item({"visualization": _headline(), "date_filter_from_context": True}) + backend = MagicMock() + backend.ask.return_value = _chat() + report = _run_one_item(item, backend, runs=1) + backend.ask.assert_called_once() + assert report.error is None + assert report.pass_at_k is True + + +def _agentic_client(chat: ChatResult) -> MagicMock: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = chat + return client + + +def _run( + chat: ChatResult, options: ExpectationOptions | None, user_context: object = _USER_CONTEXT +) -> AgenticRunSummary: + with patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(chat)): + return run_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=user_context, + options=options, + ) + + +@pytest.mark.parametrize( + ("options", "passes"), + [(None, False), (ExpectationOptions(), False), (ExpectationOptions(date_filter_from_context=_ANY), True)], +) +def test_agentic_run(options: ExpectationOptions | None, passes: bool) -> None: + ev = _run(_chat(), options).best.eval_result + assert ev.strict_pass is passes + assert ev.filter_date_score is passes + assert ev.filter_date_inherited is passes + + +def test_agentic_run_refuses_the_key_before_any_request() -> None: + client = _agentic_client(_chat()) + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + pytest.raises(ValueError, match="no date filter"), + ): + run_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="q", + expected_outputs=[_viz()], + k=1, + user_context=None, + options=ExpectationOptions(date_filter_from_context=_ANY), + ) + client.create_conversation.assert_not_called() + + +def _summary_message(chat: ChatResult, options: ExpectationOptions | None) -> str: + with ( + patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(chat)), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=_USER_CONTEXT, + options=options, + ) + return str(exc_info.value) + + +def test_strict_check_summary_shows_the_matched_dashboard_filter() -> None: + message = _summary_message(_chat(metric="metric/revenue"), ExpectationOptions(date_filter_from_context=_ANY)) + assert " date : True (matches the dashboard date filter in user_context)\n" in message + assert "Metrics Correct : False" in message + assert "context_date_filter" not in message + + +def test_strict_check_summary_is_unchanged_without_the_key() -> None: + message = _summary_message(_chat(), None) + assert " date : False\n" in message + assert "dashboard date filter" not in message + + +def test_failed_checks_do_not_blame_the_matched_date_filter() -> None: + attribute = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["x"]}} + chat = _chat(filter_by={"f_transaction_date": _INHERITED, "a": attribute}) + run = _run(chat, ExpectationOptions(date_filter_from_context=_ANY)).best + assert run.eval_result.filter_date_inherited is True + assert run.eval_result.filters_correct is False + parts = [c.get("part") for c in agentic_visualization._failed_checks(run) if c.get("name") == "filters_correct"] + assert "date" not in parts + assert "attribute" in parts