diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfda520f4..158c31187 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -752,6 +752,9 @@ Item-level keys, all optional except the candidates: `{"metric/net_sales": {"divide_by": 1000000, "decimals": 1, "suffix": "M"}}`. - `requires_visualization_part` — the chart must come back as a part of the answer, not only as the arguments of the chart tool call. +- `date_filter_from_context` — `true` or `{"dataset": ""}`: a candidate without a + date filter also accepts the chart carrying a date filter of the dashboard in the item's + `user_context`. See *Date filter carried over from the dashboard* below. Switches on a candidate chart: @@ -902,6 +905,44 @@ In a Langfuse dataset, put it in the item `metadata` (or in the `input` object) `null` value in `input` falls back to `metadata`. The value used must be an object or `null`; anything else fails the dataset load. +#### Date filter carried over from the dashboard + +A user who asks for a new chart from an open dashboard without naming a period may get the +dashboard's date filter on the chart, and the product accepts that as well as a chart with no +date filter. An item says so with `date_filter_from_context`: + +```json +"expected_output": { + "visualization": {"type": "headline_chart", "query": {"fields": {"m": {"using": "metric/gross_margin_pct"}}}, "metrics": ["m"]}, + "date_filter_from_context": {"dataset": "dataset/transaction_date"} +} +``` + +Each candidate without a date filter then also counts once more per date filter in +`user_context.view.dashboard.filters`, with that filter added. Both sides resolve against the +same day, so a relative and an absolute period that cover the same days match. The date dataset +of the added filter is, in order: the `dataset` the key states; the one the dashboard filter +names in `using` (a bare id such as `order_date` reads as `dataset/order_date`); for the +dashboard's common date filter, which names none, the one the chart's only date filter uses. +`true` therefore accepts any date dataset for the common date filter. The key's `dataset` must +be a bare id or a `dataset/` URI; anything else, such as `label/region` or a misspelled +`datset/order_date`, is an error. A dashboard filter whose `using` is not a date dataset adds +no candidate. + +A candidate that states a date filter is not extended: a question that names a period is scored +on that period only. A chart with two date filters matches no added candidate. Ranking and +attribute filters are unaffected. + +When the chart matches an added candidate, the run detail carries +`filter_date_context: {"inherited": true, "dataset": ...}` and a failure summary shows +`date : True (matches the dashboard date filter in user_context)`. The expected output reported +is still the candidate the item states; `expected_filters` shows the dashboard filter it was +scored with. A stated candidate that matched the date wins over an added one it ties with. Without the key an item is +scored exactly as before, so an item that must reject the dashboard's period ("all time") +simply leaves it out. The key on an item whose `user_context` has no dashboard date filter is +an error, raised before any request. Applies to `visualization`, `vis_agentic` and +`agentic_visualization` items; `agentic_conversation` turns do not read it. + `agentic_conversation` items take the item's `user_context` as the context of the first turn. A turn in `expected_output.turns` can set its own `user_context`, which applies from that turn on until another turn changes it, as the attached context does in the UI: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 87119eb89..b6ca2359a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -47,6 +47,7 @@ ExpectationOptions, _check_visualization_skill_activated, _evaluate_against_candidates, + check_context_date_filter, evaluation_result_detail, with_execution, with_options, @@ -213,6 +214,7 @@ def _execute_single_run( max_iterations: int = _DEFAULT_MAX_ITERATIONS, requires_execution: bool = False, options: ExpectationOptions | None = None, + user_context: dict | None = None, ) -> RunResult: """Drive one full multi-turn conversation and evaluate the result.""" options = _effective_options(options, requires_execution) @@ -314,7 +316,13 @@ def _execute_single_run( if current_result.created_visualizations and current_result.created_visualizations.objects: actual_output = current_result.created_visualizations.objects[0] - eval_result, best_expected = _evaluate_against_candidates(expected_outputs, actual_output, skill_activated) + eval_result, best_expected = _evaluate_against_candidates( + expected_outputs, + actual_output, + skill_activated, + user_context=user_context, + context_date_filter=options.date_filter_from_context, + ) eval_result = with_execution( eval_result, all_tool_call_events, current_result.text_response, options.requires_execution ) @@ -379,6 +387,8 @@ def run_agentic_visualization( ValueError: ``expected_outputs`` and ``options.expects_visualization`` disagree. An empty candidate list is only right for an item that expects no chart, and the scorer would otherwise read the default ``expects_visualization`` and fail it. + Also raised, before any request, for ``date_filter_from_context`` on an item whose + ``user_context`` has no dashboard date filter (``check_context_date_filter``). """ expects_visualization = (options or ExpectationOptions()).expects_visualization if expects_visualization != bool(expected_outputs): @@ -386,6 +396,7 @@ def run_agentic_visualization( "expected_outputs must list the candidate charts unless options.expects_visualization is false, " f"got {len(expected_outputs)} candidate(s) with expects_visualization={expects_visualization}" ) + check_context_date_filter(options or ExpectationOptions(), user_context) client = ChatClient( host=host, token=token, @@ -401,7 +412,14 @@ def run_agentic_visualization( try: run_results.append( _execute_single_run( - client, conv_id_0, question, expected_outputs, max_iterations, requires_execution, options + client, + conv_id_0, + question, + expected_outputs, + max_iterations, + requires_execution, + options, + user_context=user_context, ) ) finally: @@ -413,7 +431,14 @@ def run_agentic_visualization( try: run_results.append( _execute_single_run( - client, conv_id, question, expected_outputs, max_iterations, requires_execution, options + client, + conv_id, + question, + expected_outputs, + max_iterations, + requires_execution, + options, + user_context=user_context, ) ) finally: @@ -789,7 +814,8 @@ def _write_scores(ctx: RunTraceContext) -> None: f" expected : {sorted(ev.expected_dim_uris)}\n" f" actual : {sorted(ev.actual_dim_uris)}\n" f" Filters Correct : {ev.filters_correct}\n" - f" date : {ev.filter_date_score}\n" + f" date : {ev.filter_date_score}" + f"{' (matches the dashboard date filter in user_context)' if ev.filter_date_inherited else ''}\n" f"{_filter_diff('date', ev)}" f" ranking : {ev.filter_ranking_score}\n" f"{_filter_diff('ranking', ev)}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py index 10806d047..a0265ad12 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/visualization.py @@ -83,6 +83,15 @@ class EvaluationResult: format_correct: bool | None = None format_problems: list[str] = field(default_factory=list) part_present: bool | None = None + # Set only when the best candidate is one `date_filter_from_context` derived from the + # dashboard in the item's `user_context` and its date filter matched: the date dataset + # that filter used. Reported in the detail only when set. + filter_date_inherited_dataset: str | None = None + + @property + def filter_date_inherited(self) -> bool: + """Whether the date filter matched one taken from the dashboard in `user_context`.""" + return self.filter_date_inherited_dataset is not None @property def scored_without_chart(self) -> bool: @@ -248,6 +257,19 @@ def requires_execution_of(expected_output: object) -> bool: return isinstance(expected_output, dict) and expected_output.get("requires_execution") is True +@dataclass(frozen=True) +class ContextDateFilter: + """The item-level ``date_filter_from_context`` key. + + Every candidate without a date filter also accepts the chart carrying one of the date + filters of the dashboard in the item's ``user_context``. ``dataset`` is the date dataset + that filter must use; None takes the one the dashboard filter names, and for the + dashboard's common date filter, which names none, the one the chart used. + """ + + dataset: str | None = None + + @dataclass(frozen=True) class ExpectationOptions: """The item-level keys of a visualization expectation, beside its candidate charts. @@ -266,6 +288,10 @@ class ExpectationOptions: - ``format_intents``: per metric uri, how its numbers must read (see ``number_format``). - ``requires_visualization_part``: the chart must come back as a part of the answer. Without it, a chart that exists only in the tool arguments is scored like one the user saw. + - ``date_filter_from_context``: the chart may carry the date filter of the dashboard in the + item's ``user_context`` instead of none (see ``ContextDateFilter``). For a question that + names no period asked from an open dashboard, where carrying the dashboard's period over + and leaving the chart unfiltered are both correct. """ requires_execution: bool = False @@ -275,6 +301,69 @@ class ExpectationOptions: answer_must_not_match: tuple[str, ...] = () format_intents: Mapping[str, Mapping[str, Any] | None] | None = None requires_visualization_part: bool = False + date_filter_from_context: ContextDateFilter | None = None + + +_DATASET_PREFIX = "dataset/" + + +def date_dataset_uri(dataset: object) -> str | None: + """``dataset`` as a ``dataset/`` URI; a bare id, as dashboard filters name it, gets the prefix. + + None for anything else: not a non-empty string, or a URI of another type such as + ``label/region`` or a misspelled ``datset/order_date``. + """ + if not isinstance(dataset, str) or not dataset: + return None + if dataset.startswith(_DATASET_PREFIX): + return dataset if len(dataset) > len(_DATASET_PREFIX) else None + return None if "/" in dataset else f"{_DATASET_PREFIX}{dataset}" + + +def _context_date_filter(expected_output: dict) -> ContextDateFilter | None: + raw = expected_output.get("date_filter_from_context", False) + if raw is False: + return None + if raw is True: + return ContextDateFilter() + if isinstance(raw, dict) and set(raw) == {"dataset"}: + dataset = date_dataset_uri(raw["dataset"]) + if dataset is not None: + return ContextDateFilter(dataset=dataset) + raise ValueError( + 'date_filter_from_context must be true, false or {"dataset": ""} with a bare date dataset id ' + f"or a dataset/ URI, got {raw!r}" + ) + + +def dashboard_date_filters(user_context: object) -> list[dict]: + """The date filters of the dashboard open in ``user_context`` (``view.dashboard.filters``). + + Anything not shaped like that -- no context, no dashboard, a non-list ``filters``, an + entry that is not a ``date_filter`` object -- contributes nothing rather than raising: + the context is the chat API's schema and the dataset loader does not validate it. + """ + if not isinstance(user_context, Mapping): + return [] + view = user_context.get("view") + dashboard = view.get("dashboard") if isinstance(view, Mapping) else None + filters = dashboard.get("filters") if isinstance(dashboard, Mapping) else None + if not isinstance(filters, list): + return [] + return [f for f in filters if isinstance(f, dict) and f.get("type") == "date_filter"] + + +def check_context_date_filter(options: ExpectationOptions, user_context: object) -> None: + """Refuse ``date_filter_from_context`` on an item whose context has no dashboard date filter. + + Raises: + ValueError: the key is set and ``user_context`` offers nothing to take a date filter + from, so the key would silently accept nothing. + """ + if options.date_filter_from_context is not None and not dashboard_date_filters(user_context): + raise ValueError( + "date_filter_from_context is set, but the item's user_context has no date filter in view.dashboard.filters" + ) def _regex_error(pattern: str) -> str | None: @@ -308,6 +397,7 @@ def _patterns(expected_output: dict, key: str) -> tuple[str, ...]: "answer_must_not_match", "format_intents", "requires_visualization_part", + "date_filter_from_context", } ) @@ -367,6 +457,9 @@ def expectation_options(expected_output: object) -> ExpectationOptions: validate_format_intents(intents) if not expects and (intents is not None or part): raise ValueError("an expectation without a chart cannot state format_intents or requires_visualization_part") + context_date = _context_date_filter(expected_output) + if not expects and context_date is not None: + raise ValueError("an expectation without a chart cannot state date_filter_from_context") return ExpectationOptions( requires_execution=requires_execution_of(expected_output), expects_visualization=expects, @@ -375,6 +468,7 @@ def expectation_options(expected_output: object) -> ExpectationOptions: answer_must_not_match=_patterns(expected_output, "answer_must_not_match"), format_intents=intents, requires_visualization_part=part, + date_filter_from_context=context_date, ) @@ -551,17 +645,93 @@ def _evaluate_against_candidates( actual: CreatedVisualization | None, skill_activated: bool = False, today: date | None = None, + *, + user_context: object = None, + context_date_filter: ContextDateFilter | None = None, ) -> tuple[EvaluationResult, CreatedVisualization | None]: + """Score ``actual`` against every candidate and return the best result with its candidate. + + The returned candidate is always one the item states. With ``context_date_filter``, the + candidates ``context_date_candidates`` derives are scored after the stated ones and can + replace the best stated result only when they rank higher, or when they tie with it and + match the date it misses. Among stated candidates that tie, one that matched the date is + that best result -- so a chart failing on something else is not also blamed for + the dashboard's period, and a stated candidate that matched the date keeps winning. The + result is then marked with the date dataset, and the candidate returned is the stated one + the winner was derived from. + """ if not expected_outputs: return _no_candidate_result(actual, skill_activated), None # Anchored once here too: the candidates are ranked against each other, so letting # each resolve its own today could rank them on spans from different days. today = today or date.today() - pairs = [(_evaluate_visualization(exp, actual, skill_activated, today), exp) for exp in expected_outputs] - best_result, best_expected = max(pairs, key=lambda p: (p[0].strict_pass, p[0].strict_checks_passed_count)) + stated = [(_evaluate_visualization(exp, actual, skill_activated, today), exp) for exp in expected_outputs] + if context_date_filter is None: + return max(stated, key=lambda pair: _rank(pair[0])) + # With the key, a stated candidate that matched the date wins the stated ones it ties + # with, so the derived candidates below are measured against it. + best_result, best_expected = max(stated, key=lambda pair: (*_rank(pair[0]), pair[0].filter_date_score)) + for derived, origin, dataset in context_date_candidates( + expected_outputs, actual, user_context, context_date_filter + ): + result = _evaluate_visualization(derived, actual, skill_activated, today) + if not result.filter_date_score: + continue + ranks_higher = _rank(result) > _rank(best_result) + fixes_the_date = _rank(result) == _rank(best_result) and not best_result.filter_date_score + if ranks_higher or fixes_the_date: + best_result = replace(result, filter_date_inherited_dataset=dataset) + best_expected = origin return best_result, best_expected +def _rank(result: EvaluationResult) -> tuple[bool, int]: + return result.strict_pass, result.strict_checks_passed_count + + +def _date_filters(viz: CreatedVisualization) -> list[dict]: + return [f for f in viz.query.filter_by.values() if f.get("type") == "date_filter"] + + +def context_date_candidates( + expected_outputs: list[CreatedVisualization], + actual: CreatedVisualization | None, + user_context: object, + context_date_filter: ContextDateFilter, +) -> list[tuple[CreatedVisualization, CreatedVisualization, str]]: + """Each candidate without a date filter, once per dashboard date filter in ``user_context``. + + Every copy carries that dashboard filter, on the date dataset ``ContextDateFilter`` picks, + and is returned with the candidate it was copied from and that dataset. A dashboard + filter naming no dataset, with no dataset stated on the item, takes the dataset of the + chart's only date filter; with no such filter, or one whose ``using`` is not a ``dataset/`` + URI, there is nothing for it to match, and it yields no copy. A dashboard filter whose + ``using`` is not a date dataset yields none either. + """ + actual_dates = _date_filters(actual) if actual is not None else [] + actual_using = actual_dates[0].get("using") if len(actual_dates) == 1 else None + actual_dataset = actual_using if date_dataset_uri(actual_using) == actual_using else None + derived: list[tuple[CreatedVisualization, CreatedVisualization, str]] = [] + for expected in expected_outputs: + if _date_filters(expected): + continue + for dashboard_filter in dashboard_date_filters(user_context): + dashboard_using = dashboard_filter.get("using") + dashboard_dataset = date_dataset_uri(dashboard_using) + if dashboard_using and dashboard_dataset is None: + continue + dataset = context_date_filter.dataset or dashboard_dataset or actual_dataset + if dataset is None: + continue + key = "context_date_filter" + while key in expected.query.filter_by: + key = f"_{key}" + filter_by = {**expected.query.filter_by, key: {**dashboard_filter, "using": dataset}} + query = expected.query.model_copy(update={"filter_by": filter_by}) + derived.append((expected.model_copy(update={"query": query}), expected, dataset)) + return derived + + def _parse_expected(expected_output: dict) -> list[CreatedVisualization]: if not isinstance(expected_output, dict): raise ValueError("'expected_output' must be a JSON object") @@ -624,6 +794,13 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict: "actual_dim_uris": sorted(ev.actual_dim_uris), "expected_filters": ev.expected_filters, "actual_filters": ev.actual_filters, + # Nested and only when it applies: `quality_score` counts every top-level boolean, and + # an item without `date_filter_from_context` must not carry the key at all. + **( + {"filter_date_context": {"inherited": True, "dataset": ev.filter_date_inherited_dataset}} + if ev.filter_date_inherited + else {} + ), "expected_sorts": ev.expected_sorts, "actual_sorts": ev.actual_sorts, **({"metric_layout_problems": ev.metric_layout_problems} if ev.metric_layout_problems else {}), @@ -653,12 +830,29 @@ def evaluation_result_detail(ev: EvaluationResult) -> dict: class VisualizationEvaluator: test_kind = "visualization" + def validate(self, item: DatasetItem) -> None: + """Refuse a malformed expectation before the runner sends the item's question. + + Raises: + ValueError: an item-level key or a candidate chart is malformed, or + ``date_filter_from_context`` has no dashboard date filter to take a filter from. + """ + check_context_date_filter(expectation_options(item.expected_output), item.user_context) + _parse_expected(item.expected_output) + def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation: options = expectation_options(item.expected_output) + check_context_date_filter(options, item.user_context) candidates = _parse_expected(item.expected_output) actual = _extract_actual(chat_result) skill_activated = _check_visualization_skill_activated(chat_result.tool_call_events) - ev, _best_expected = _evaluate_against_candidates(candidates, actual, skill_activated) + ev, _best_expected = _evaluate_against_candidates( + candidates, + actual, + skill_activated, + user_context=item.user_context, + context_date_filter=options.date_filter_from_context, + ) ev = with_execution(ev, chat_result.tool_call_events, chat_result.text_response, options.requires_execution) ev = with_options(ev, options, actual, chat_result.tool_call_events, render_answer_text(chat_result)) return ItemEvaluation( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index c93488faa..b0c8d0a43 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -295,7 +295,12 @@ def _run_one_item( best_chat_result: ChatResult | None = None best_run_latency: float | None = None judge_errors: list[str] = [] + # Optional on the Evaluator protocol: an evaluator that can tell a malformed item from its + # expectation alone refuses it here, before the item's question is sent to the agent. + validate = getattr(evaluator, "validate", None) try: + if validate is not None: + validate(item) for run_index in range(1, runs + 1): t0 = time.perf_counter() chat_result = backend.ask(item) diff --git a/packages/gooddata-eval/tests/test_context_date_filter.py b/packages/gooddata-eval/tests/test_context_date_filter.py new file mode 100644 index 000000000..080603348 --- /dev/null +++ b/packages/gooddata-eval/tests/test_context_date_filter.py @@ -0,0 +1,547 @@ +# (C) 2026 GoodData Corporation +"""The item-level `date_filter_from_context` key: a date filter taken from the dashboard in `user_context`. + +Without the key, every case here must score exactly as the stated candidates alone do. +""" + +import json +from datetime import date +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.core.agentic import visualization as agentic_visualization +from gooddata_eval.core.agentic.visualization import ( + AgenticRunSummary, + VisualizationAssertionError, + evaluate_agentic_visualization, + run_agentic_visualization, +) +from gooddata_eval.core.evaluators import get_evaluator +from gooddata_eval.core.evaluators.visualization import ( + ContextDateFilter, + EvaluationResult, + ExpectationOptions, + _evaluate_against_candidates, + check_context_date_filter, + context_date_candidates, + dashboard_date_filters, + date_dataset_uri, + evaluation_result_detail, + expectation_options, +) +from gooddata_eval.core.models import ChatResult, CreatedVisualization, DatasetItem +from gooddata_eval.core.runner import _run_one_item + +_TODAY = date(2026, 11, 12) + +# The dashboard of globalmart_visualization_charts_ctx: a common date filter, no dataset named. +_USER_CONTEXT = { + "view": { + "dashboard": { + "id": "dashboard_033", + "title": "Order Lines & Margin", + "filters": [{"type": "date_filter", "granularity": "MONTH", "from": -11, "to": 0}], + } + } +} +_INHERITED = { + "type": "date_filter", + "using": "dataset/transaction_date", + "granularity": "MONTH", + "from": -11, + "to": 0, +} +_ANY = ContextDateFilter() +_TRANSACTION_DATE = ContextDateFilter(dataset="dataset/transaction_date") + + +def _headline(filter_by: dict | None = None, metric: str = "metric/gross_margin_percent") -> dict: + return { + "id": "v", + "type": "headline_chart", + "query": {"fields": {"m": {"using": metric}}, "filter_by": filter_by or {}}, + "metrics": ["m"], + } + + +def _viz(filter_by: dict | None = None) -> CreatedVisualization: + return CreatedVisualization.model_validate(_headline(filter_by)) + + +def _date(frm: int, to: int, granularity: str = "MONTH", using: str = "dataset/transaction_date") -> dict: + return {"type": "date_filter", "using": using, "granularity": granularity, "from": frm, "to": to} + + +def _context(*filters: dict) -> dict: + return {"view": {"dashboard": {"id": "d", "filters": list(filters)}}} + + +def _score( + actual_filter_by: dict | None, + expected_filter_by: dict | None = None, + key: ContextDateFilter | None = _ANY, + user_context: object = _USER_CONTEXT, +) -> EvaluationResult: + ev, _ = _evaluate_against_candidates( + [_viz(expected_filter_by)], + _viz(actual_filter_by), + True, + _TODAY, + user_context=user_context, + context_date_filter=key, + ) + return ev + + +# ── parsing the key ────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("raw", "parsed"), + [ + (False, None), + (True, _ANY), + ({"dataset": "dataset/transaction_date"}, _TRANSACTION_DATE), + ({"dataset": "transaction_date"}, _TRANSACTION_DATE), + ], +) +def test_the_key_parses(raw: object, parsed: ContextDateFilter | None) -> None: + options = expectation_options({"visualization": _headline(), "date_filter_from_context": raw}) + assert options.date_filter_from_context == parsed + + +def test_an_absent_key_parses_to_none() -> None: + assert expectation_options({"visualization": _headline()}).date_filter_from_context is None + + +@pytest.mark.parametrize( + "raw", + [ + None, + "yes", + 1, + {}, + {"dataset": ""}, + {"dataset": 1}, + {"dataset": "x", "extra": 1}, + {"dataset": "datset/transaction_date"}, + {"dataset": "label/region"}, + {"dataset": "dataset/"}, + ], +) +def test_a_malformed_key_is_refused(raw: object) -> None: + with pytest.raises(ValueError, match="date_filter_from_context"): + expectation_options({"visualization": _headline(), "date_filter_from_context": raw}) + + +def test_the_key_on_an_item_without_a_chart_is_refused() -> None: + with pytest.raises(ValueError, match="date_filter_from_context"): + expectation_options({"expects_visualization": False, "date_filter_from_context": True}) + + +@pytest.mark.parametrize("user_context", [None, {}, {"view": {}}, {"view": {"dashboard": {"id": "d"}}}, "text"]) +def test_the_key_without_a_dashboard_date_filter_in_context_is_refused(user_context: object) -> None: + with pytest.raises(ValueError, match="no date filter"): + check_context_date_filter(ExpectationOptions(date_filter_from_context=_ANY), user_context) + + +def test_without_the_key_any_context_is_fine() -> None: + check_context_date_filter(ExpectationOptions(), None) + + +@pytest.mark.parametrize( + ("raw", "uri"), + [ + ("order_date", "dataset/order_date"), + ("dataset/order_date", "dataset/order_date"), + ("datset/order_date", None), + ("label/region", None), + ("dataset/", None), + ("", None), + (None, None), + ], +) +def test_date_dataset_uri(raw: object, uri: str | None) -> None: + assert date_dataset_uri(raw) == uri + + +@pytest.mark.parametrize("malformed", ["text", {"view": "x"}, {"view": {"dashboard": {"filters": "x"}}}]) +def test_a_malformed_context_yields_no_dashboard_filter(malformed: object) -> None: + assert dashboard_date_filters(malformed) == [] + + +# ── scoring with the key ───────────────────────────────────────────────────── + + +def test_the_dashboard_period_passes_and_is_marked_inherited() -> None: + ev = _score({"f_transaction_date": _INHERITED}) + assert ev.strict_pass is True + assert ev.filter_date_score is True + assert ev.filter_date_inherited_dataset == "dataset/transaction_date" + + +def test_no_date_filter_still_passes_without_being_inherited() -> None: + ev = _score({}) + assert ev.strict_pass is True + assert ev.filter_date_inherited is False + + +def test_a_period_other_than_the_dashboards_fails() -> None: + ev = _score({"f": _date(-5, 0)}) + assert ev.filter_date_score is False + assert ev.filter_date_inherited is False + + +@pytest.mark.parametrize( + ("expected_filter", "actual_filter", "date_ok"), + [ + (_date(-1, -1, "YEAR"), _INHERITED, False), + (_date(-1, -1, "YEAR"), _date(-1, -1, "YEAR"), True), + (_INHERITED, _INHERITED, True), + ], +) +def test_a_candidate_stating_a_date_filter_is_not_extended( + expected_filter: dict, actual_filter: dict, date_ok: bool +) -> None: + ev = _score({"f": actual_filter}, {"f": expected_filter}) + assert ev.filter_date_score is date_ok + assert ev.filter_date_inherited is False + + +def test_the_common_filter_with_true_takes_the_charts_dataset() -> None: + ev = _score({"f": _date(-11, 0, using="dataset/order_date")}) + assert ev.filter_date_score is True + assert ev.filter_date_inherited_dataset == "dataset/order_date" + + +@pytest.mark.parametrize( + ("actual_dataset", "date_ok"), [("dataset/transaction_date", True), ("dataset/ship_date", False)] +) +def test_a_stated_dataset_accepts_only_that_one(actual_dataset: str, date_ok: bool) -> None: + ev = _score({"f": _date(-11, 0, using=actual_dataset)}, key=_TRANSACTION_DATE) + assert ev.filter_date_score is date_ok + assert ev.filter_date_inherited is date_ok + + +@pytest.mark.parametrize("dashboard_using", ["order_date", "dataset/order_date"]) +@pytest.mark.parametrize(("actual_dataset", "date_ok"), [("dataset/order_date", True), ("dataset/ship_date", False)]) +def test_a_dashboard_filter_naming_a_dataset_accepts_only_that_one( + dashboard_using: str, actual_dataset: str, date_ok: bool +) -> None: + context = _context({"type": "date_filter", "using": dashboard_using, "granularity": "MONTH", "from": -11, "to": 0}) + ev = _score({"f": _date(-11, 0, using=actual_dataset)}, user_context=context) + assert ev.filter_date_score is date_ok + + +def test_a_stated_dataset_wins_over_the_dashboard_filters() -> None: + context = _context(_date(-11, 0, using="dataset/order_date")) + ev = _score({"f": _INHERITED}, key=_TRANSACTION_DATE, user_context=context) + assert ev.filter_date_inherited_dataset == "dataset/transaction_date" + + +def test_a_chart_date_filter_naming_no_dataset_matches_nothing() -> None: + nameless = {k: v for k, v in _date(-11, 0).items() if k != "using"} + ev = _score({"f": nameless}) + assert ev.filter_date_score is False + + +def test_the_same_span_written_absolutely_is_accepted() -> None: + absolute = {"type": "date_filter", "using": "dataset/transaction_date", "from": "2025-12-01", "to": "2026-11-30"} + ev = _score({"f": absolute}) + assert ev.filter_date_score is True + assert ev.filter_date_inherited is True + + +def test_two_chart_date_filters_fail_even_if_one_is_the_dashboards() -> None: + ev = _score({"a": _INHERITED, "b": _date(-1, -1, "YEAR", using="dataset/order_date")}) + assert ev.filter_date_score is False + assert ev.filter_date_inherited is False + + +def test_any_of_several_dashboard_date_filters_matches() -> None: + context = _context( + {"type": "attribute_filter", "using": "label/region"}, + {"type": "date_filter", "granularity": "YEAR", "from": 0, "to": 0}, + {"type": "date_filter", "granularity": "MONTH", "from": -11, "to": 0}, + ) + assert len(dashboard_date_filters(context)) == 2 + assert _score({"f": _INHERITED}, user_context=context).filter_date_score is True + + +def test_ranking_and_attribute_filters_are_not_relaxed() -> None: + attribute = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}} + ev = _score({"f": _INHERITED, "r": attribute}) + assert ev.filter_date_score is True + assert ev.filter_attribute_score is False + assert ev.strict_pass is False + + +def test_derived_candidates_keep_the_stated_filters() -> None: + attribute = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}} + expected = _viz({"context_date_filter": attribute}) + derived = context_date_candidates([expected], _viz({"f": _INHERITED}), _USER_CONTEXT, _ANY) + assert len(derived) == 1 + candidate, origin, dataset = derived[0] + assert origin is expected + assert dataset == "dataset/transaction_date" + assert candidate.query.filter_by["context_date_filter"] == attribute + assert candidate.query.filter_by["_context_date_filter"] == _INHERITED + assert expected.query.filter_by == {"context_date_filter": attribute} + + +def test_a_dashboard_filter_whose_using_is_not_a_date_dataset_adds_no_candidate() -> None: + context = _context({"type": "date_filter", "using": "label/region", "granularity": "MONTH", "from": -11, "to": 0}) + assert context_date_candidates([_viz()], _viz({"f": _INHERITED}), context, _ANY) == [] + assert _score({"f": _INHERITED}, user_context=context).filter_date_score is False + + +def test_a_matched_derived_candidate_reports_the_stated_one() -> None: + expected = [_viz()] + ev, best = _evaluate_against_candidates( + expected, _viz({"f": _INHERITED}), True, _TODAY, user_context=_USER_CONTEXT, context_date_filter=_ANY + ) + assert best is expected[0] + assert ev.filter_date_inherited_dataset == "dataset/transaction_date" + assert len(ev.expected_filters["date"]) == 1 + + +def test_a_stated_candidate_matching_the_date_beats_a_derived_one_it_ties_with() -> None: + stated_dated, stated_plain = _viz({"f": _INHERITED}), _viz() + wrong_metric = CreatedVisualization.model_validate(_headline({"f": _INHERITED}, metric="metric/revenue")) + ev, best = _evaluate_against_candidates( + [stated_dated, stated_plain], + wrong_metric, + True, + _TODAY, + user_context=_USER_CONTEXT, + context_date_filter=_ANY, + ) + assert best is stated_dated + assert ev.filter_date_score is True + assert ev.filter_date_inherited is False + + +def test_a_stated_candidate_matching_the_date_wins_the_stated_ones_it_ties_with() -> None: + region = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["EMEA"]}} + other_region = {**region, "state": {"include": ["APAC"]}} + right_metric_wrong_region = _viz({"a": other_region}) + wrong_metric_right_filters = CreatedVisualization.model_validate( + _headline({"d": _INHERITED, "a": region}, metric="metric/revenue") + ) + candidates = [right_metric_wrong_region, wrong_metric_right_filters] + actual = _viz({"d": _INHERITED, "a": region}) + ev, best = _evaluate_against_candidates( + candidates, actual, True, _TODAY, user_context=_USER_CONTEXT, context_date_filter=_ANY + ) + assert best is wrong_metric_right_filters + assert ev.filter_date_inherited is False + _, best_without_key = _evaluate_against_candidates(candidates, actual, True, _TODAY) + assert best_without_key is right_metric_wrong_region + + +@pytest.mark.parametrize("using", ["label/region", "transaction_date", "datset/transaction_date"]) +def test_a_chart_date_filter_not_on_a_dataset_uri_matches_nothing(using: str) -> None: + ev = _score({"f": _date(-11, 0, using=using)}) + assert ev.filter_date_score is False + assert ev.strict_pass is False + assert ev.filter_date_inherited is False + + +def test_the_stated_candidate_is_returned_when_it_ties() -> None: + expected = [_viz()] + _, best = _evaluate_against_candidates( + expected, _viz(), True, _TODAY, user_context=_USER_CONTEXT, context_date_filter=_ANY + ) + assert best is expected[0] + + +# ── without the key nothing changes ────────────────────────────────────────── + + +@pytest.mark.parametrize( + "actual_filter_by", + [{}, {"f": _INHERITED}, {"f": _date(-5, 0)}, {"a": _INHERITED, "b": _date(0, 0, "YEAR")}], +) +@pytest.mark.parametrize("expected_filter_by", [{}, {"f": _date(-1, -1, "YEAR")}]) +def test_without_the_key_the_detail_is_byte_identical(expected_filter_by: dict, actual_filter_by: dict) -> None: + expected, actual = [_viz(expected_filter_by)], _viz(actual_filter_by) + before, _ = _evaluate_against_candidates(expected, actual, True, _TODAY) + after, _ = _evaluate_against_candidates(expected, actual, True, _TODAY, user_context=_USER_CONTEXT) + assert after == before + dump = json.dumps(evaluation_result_detail(before), sort_keys=True, default=sorted) + assert json.dumps(evaluation_result_detail(after), sort_keys=True, default=sorted) == dump + assert "filter_date_context" not in dump + + +# ── the item from the ticket, through both evaluation paths ───────────────── + + +def _item(expected_output: dict) -> DatasetItem: + return DatasetItem( + id="globalmart_visualization_charts_ctx_2", + dataset_name="globalmart_visualization_charts_ctx", + test_kind="visualization", + question="Show Gross Margin Percent as a headline.", + expected_output=expected_output, + user_context=_USER_CONTEXT, + ) + + +def _chat(metric: str = "metric/gross_margin_percent", filter_by: dict | None = None) -> ChatResult: + return ChatResult.model_validate( + { + "createdVisualizations": { + "objects": [_headline(filter_by or {"f_transaction_date": _INHERITED}, metric=metric)], + "reasoning": "", + }, + "toolCallEvents": [], + } + ) + + +def test_single_shot_fails_without_the_key() -> None: + result = get_evaluator("visualization").evaluate(_item({"visualization": _headline()}), _chat()) + assert result.passed is False + assert result.detail["filter_date_score"] is False + assert "filter_date_context" not in result.detail + + +def test_single_shot_passes_with_the_key() -> None: + item = _item({"visualization": _headline(), "date_filter_from_context": True}) + result = get_evaluator("visualization").evaluate(item, _chat()) + assert result.passed is True + assert result.detail["filter_date_score"] is True + assert result.detail["filter_date_context"] == {"inherited": True, "dataset": "dataset/transaction_date"} + + +def test_single_shot_refuses_the_key_without_a_dashboard_date_filter() -> None: + item = _item({"visualization": _headline(), "date_filter_from_context": True}) + item = item.model_copy(update={"user_context": None}) + with pytest.raises(ValueError, match="no date filter"): + get_evaluator("visualization").evaluate(item, _chat()) + + +@pytest.mark.parametrize( + ("expected_output", "user_context", "error"), + [ + ({"visualization": _headline(), "date_filter_from_context": True}, None, "no date filter"), + ({"visualization": _headline(), "date_filter_from_context": {"dataset": "label/x"}}, _USER_CONTEXT, "dataset"), + ({"visualization": _headline(), "answer_must_inlcude": ["x"]}, _USER_CONTEXT, "answer_must_inlcude"), + ], +) +def test_single_shot_runner_refuses_a_malformed_item_before_asking( + expected_output: dict, user_context: dict | None, error: str +) -> None: + item = _item(expected_output).model_copy(update={"user_context": user_context}) + backend = MagicMock() + report = _run_one_item(item, backend, runs=2) + backend.ask.assert_not_called() + assert report.runs == 0 + assert report.error is not None + assert report.error.startswith("ValueError:") + assert error in report.error + + +def test_single_shot_runner_asks_for_a_valid_item() -> None: + item = _item({"visualization": _headline(), "date_filter_from_context": True}) + backend = MagicMock() + backend.ask.return_value = _chat() + report = _run_one_item(item, backend, runs=1) + backend.ask.assert_called_once() + assert report.error is None + assert report.pass_at_k is True + + +def _agentic_client(chat: ChatResult) -> MagicMock: + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = chat + return client + + +def _run( + chat: ChatResult, options: ExpectationOptions | None, user_context: object = _USER_CONTEXT +) -> AgenticRunSummary: + with patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(chat)): + return run_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=user_context, + options=options, + ) + + +@pytest.mark.parametrize( + ("options", "passes"), + [(None, False), (ExpectationOptions(), False), (ExpectationOptions(date_filter_from_context=_ANY), True)], +) +def test_agentic_run(options: ExpectationOptions | None, passes: bool) -> None: + ev = _run(_chat(), options).best.eval_result + assert ev.strict_pass is passes + assert ev.filter_date_score is passes + assert ev.filter_date_inherited is passes + + +def test_agentic_run_refuses_the_key_before_any_request() -> None: + client = _agentic_client(_chat()) + with ( + patch.object(agentic_visualization, "ChatClient", return_value=client), + pytest.raises(ValueError, match="no date filter"), + ): + run_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="q", + expected_outputs=[_viz()], + k=1, + user_context=None, + options=ExpectationOptions(date_filter_from_context=_ANY), + ) + client.create_conversation.assert_not_called() + + +def _summary_message(chat: ChatResult, options: ExpectationOptions | None) -> str: + with ( + patch.object(agentic_visualization, "ChatClient", return_value=_agentic_client(chat)), + pytest.raises(VisualizationAssertionError) as exc_info, + ): + evaluate_agentic_visualization( + host="https://example.com", + token="tok", + workspace_id="ws", + question="Show Gross Margin Percent as a headline.", + expected_outputs=[_viz()], + k=1, + user_context=_USER_CONTEXT, + options=options, + ) + return str(exc_info.value) + + +def test_strict_check_summary_shows_the_matched_dashboard_filter() -> None: + message = _summary_message(_chat(metric="metric/revenue"), ExpectationOptions(date_filter_from_context=_ANY)) + assert " date : True (matches the dashboard date filter in user_context)\n" in message + assert "Metrics Correct : False" in message + assert "context_date_filter" not in message + + +def test_strict_check_summary_is_unchanged_without_the_key() -> None: + message = _summary_message(_chat(), None) + assert " date : False\n" in message + assert "dashboard date filter" not in message + + +def test_failed_checks_do_not_blame_the_matched_date_filter() -> None: + attribute = {"type": "attribute_filter", "using": "label/region", "state": {"include": ["x"]}} + chat = _chat(filter_by={"f_transaction_date": _INHERITED, "a": attribute}) + run = _run(chat, ExpectationOptions(date_filter_from_context=_ANY)).best + assert run.eval_result.filter_date_inherited is True + assert run.eval_result.filters_correct is False + parts = [c.get("part") for c in agentic_visualization._failed_checks(run) if c.get("name") == "filters_correct"] + assert "date" not in parts + assert "attribute" in parts