diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_metric_expectation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_metric_expectation.py new file mode 100644 index 000000000..990bde9ef --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_metric_expectation.py @@ -0,0 +1,434 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""What a metric-skill item expects, parsed from its ``expected_output``, and the matchers that judge it. + +An item states its expectation in one of two shapes. A list of ``{"maql": ...}`` candidates, or a +dict carrying one candidate, is the original shape: the agent must create one metric whose MAQL +matches a candidate. A dict that carries any key of ``OPTION_KEYS`` is the options shape: every key +is a check the item opts into, and an item without a key is not held to it. A malformed options dict +raises ``ValueError`` before any request, so a typo never reads green. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any + +from gooddata_eval.core.evaluators._maql import normalize_maql + +FORMAT_KINDS = ("percent", "currency", "integer", "decimal") + +_FIELD_KEYS = ( + "title", + "metric_id", + "format", + "format_kind", + "format_contains", + "description", + "description_contains", + "tags", + "tags_include", +) +_UPDATE_KEYS = (*_FIELD_KEYS, "maql") +OPTION_KEYS = frozenset( + { + "candidates", + "user_replies", + "run_all_replies", + "expect_no_create", + "expected_mentions", + "expect_no_overwrite", + "expect_clarification", + "confirmation_turn", + "expect_confirmation_stop", + "rubric", + "expected_fields", + "expected_update", + "expected_metrics", + "expect_no_deletion", + "expect_executes", + "score_first_proposal", + "expect_suggestions", + } +) + +_CURRENCY_MARKS = ("$", "€", "£", "¥", "Kč") +_DECIMAL_RE = re.compile(r"\.[0#]") +_PLACEHOLDER_RE = re.compile(r"\{metric/@(\d+)\}") +# A formula in an inline code span or in a fenced block, with or without a language tag. +_PROPOSED_MAQL_RE = re.compile(r"```[a-zA-Z]*\s*(SELECT\b[^`]*?)\s*```|`(SELECT\b[^`]*)`", re.IGNORECASE) + + +@dataclass(frozen=True) +class FieldSpec: + """Values the stored metric must carry. A field left as None or empty is not checked.""" + + title: str | None = None + metric_id: str | None = None + format: str | None = None + format_kind: str | None = None + format_contains: tuple[str, ...] = () + description: str | None = None + description_contains: tuple[str, ...] = () + tags: frozenset[str] | None = None + tags_include: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class UpdateSpec: + """An existing metric the agent must update: the listed fields change, every other field stays.""" + + metric_id: str + fields: FieldSpec + maql: tuple[str, ...] = () + + +@dataclass(frozen=True) +class MetricExpectation: + """The parsed expectation of one metric-skill item.""" + + candidates: list[dict] = field(default_factory=list) + legacy: bool = True + user_replies: tuple[str, ...] = () + run_all_replies: bool = False + expect_no_create: bool = False + expected_mentions: tuple[str, ...] = () + expect_no_overwrite: bool = False + expect_clarification: bool = False + confirmation_turn: int | None = None + rubric: str | None = None + expected_fields: FieldSpec | None = None + expected_update: UpdateSpec | None = None + expected_metrics: tuple[tuple[str, ...], ...] = () + # Ids of metrics that must still exist after the run. A list, not a flag: other items create and delete + # their own metrics in the same workspace at the same time. + expect_no_deletion: tuple[str, ...] = () + expect_executes: bool = False + score_first_proposal: bool = False + expect_suggestions: bool = False + + @property + def needs_workspace(self) -> bool: + """Whether a run reads the workspace's metrics through REST: every options-shape item does.""" + return not self.legacy + + +def _is_options_shape(raw: Any) -> bool: + return isinstance(raw, dict) and any(key in OPTION_KEYS for key in raw) + + +def _strings(value: Any, name: str) -> tuple[str, ...]: + """A non-empty string, or a list of them, as a tuple.""" + items = [value] if isinstance(value, str) else value + if not isinstance(items, list) or not items or not all(isinstance(s, str) and s.strip() for s in items): + raise ValueError(f"{name} must be a non-empty string or a list of them, got {value!r}") + return tuple(items) + + +def _flag(raw: dict, name: str) -> bool: + value = raw.get(name, False) + if not isinstance(value, bool): + raise ValueError(f"{name} must be true or false, got {value!r}") + return value + + +def _candidates(value: Any, name: str) -> list[dict]: + """``[{"maql": ...}]`` or a plain list of MAQL strings, as candidate dicts.""" + if isinstance(value, str): + value = [value] + if not isinstance(value, list) or not value: + raise ValueError(f"{name} must be a non-empty list of MAQL candidates, got {value!r}") + out: list[dict] = [] + for candidate in value: + maql = candidate.get("maql") if isinstance(candidate, dict) else candidate + if not isinstance(maql, str) or not maql.strip(): + raise ValueError(f"{name}: every candidate needs a non-empty MAQL, got {candidate!r}") + out.append(candidate if isinstance(candidate, dict) else {"maql": maql}) + return out + + +def _field_spec(raw: Any, name: str, allowed: tuple[str, ...]) -> FieldSpec: + if not isinstance(raw, dict) or not raw: + raise ValueError(f"{name} must be a non-empty object, got {raw!r}") + unknown = sorted(set(raw) - set(allowed)) + if unknown: + raise ValueError(f"{name}: unknown key(s) {unknown}; allowed: {list(allowed)}") + kind = raw.get("format_kind") + if "format_kind" in raw and kind not in FORMAT_KINDS: + raise ValueError(f"{name}.format_kind must be one of {FORMAT_KINDS}, got {kind!r}") + for key in ("title", "metric_id", "format", "description"): + if key in raw and (not isinstance(raw[key], str) or not raw[key].strip()): + raise ValueError(f"{name}.{key} must be a non-empty string, got {raw[key]!r}") + return FieldSpec( + title=raw.get("title"), + metric_id=raw.get("metric_id"), + format=raw.get("format"), + format_kind=kind, + format_contains=_strings(raw["format_contains"], f"{name}.format_contains") if "format_contains" in raw else (), + description=raw.get("description"), + description_contains=( + _strings(raw["description_contains"], f"{name}.description_contains") + if "description_contains" in raw + else () + ), + tags=frozenset(_strings(raw["tags"], f"{name}.tags")) if "tags" in raw else None, + tags_include=frozenset(_strings(raw["tags_include"], f"{name}.tags_include")) + if "tags_include" in raw + else frozenset(), + ) + + +def parse_expectation(raw: Any) -> MetricExpectation: + """Parse and validate an item's ``expected_output``. + + Raises: + ValueError: The options shape names an unknown key, a key holds the wrong type, two keys + contradict each other, or the item checks nothing at all. + """ + if not _is_options_shape(raw): + legacy = raw if isinstance(raw, list) else [raw] + if not all(isinstance(c, dict) for c in legacy): + raise ValueError( + f"expected_output: a candidate list holds dicts with a 'maql' key, got {raw!r}; " + "plain MAQL strings go under 'candidates'" + ) + return MetricExpectation(candidates=list(legacy)) + + unknown = sorted(set(raw) - OPTION_KEYS) + if unknown: + raise ValueError(f"expected_output: unknown key(s) {unknown}; allowed: {sorted(OPTION_KEYS)}") + + candidates = _candidates(raw["candidates"], "candidates") if "candidates" in raw else [] + replies = _strings(raw["user_replies"], "user_replies") if "user_replies" in raw else () + run_all = _flag(raw, "run_all_replies") + no_create = _flag(raw, "expect_no_create") + mentions = _strings(raw["expected_mentions"], "expected_mentions") if "expected_mentions" in raw else () + + if "confirmation_turn" in raw and "expect_confirmation_stop" in raw: + raise ValueError("give confirmation_turn or expect_confirmation_stop, not both") + confirmation_turn: int | None = None + if "confirmation_turn" in raw: + confirmation_turn = raw["confirmation_turn"] + if isinstance(confirmation_turn, bool) or not isinstance(confirmation_turn, int) or confirmation_turn < 2: + raise ValueError(f"confirmation_turn must be an integer of at least 2, got {confirmation_turn!r}") + elif _flag(raw, "expect_confirmation_stop"): + confirmation_turn = 2 + + rubric = raw.get("rubric") + if "rubric" in raw and (not isinstance(rubric, str) or not rubric.strip()): + raise ValueError(f"rubric must be a non-empty string, got {rubric!r}") + + fields = _field_spec(raw["expected_fields"], "expected_fields", _FIELD_KEYS) if "expected_fields" in raw else None + + update: UpdateSpec | None = None + if "expected_update" in raw: + upd = raw["expected_update"] + if not isinstance(upd, dict) or not isinstance(upd.get("metric_id"), str) or not upd["metric_id"].strip(): + raise ValueError(f"expected_update needs a metric_id, got {upd!r}") + maql = _strings(upd["maql"], "expected_update.maql") if "maql" in upd else () + rest = {k: v for k, v in upd.items() if k not in ("metric_id", "maql")} + unknown_upd = sorted(set(rest) - set(_UPDATE_KEYS)) + if unknown_upd: + raise ValueError(f"expected_update: unknown key(s) {unknown_upd}; allowed: {['metric_id', *_UPDATE_KEYS]}") + if not rest and not maql: + raise ValueError("expected_update must list at least one field that changes") + update = UpdateSpec( + metric_id=upd["metric_id"], + fields=_field_spec(rest, "expected_update", _FIELD_KEYS) if rest else FieldSpec(), + maql=maql, + ) + + metrics: tuple[tuple[str, ...], ...] = () + if "expected_metrics" in raw: + entries = raw["expected_metrics"] + if not isinstance(entries, list) or len(entries) < 2: + raise ValueError(f"expected_metrics must list at least two metrics, got {entries!r}") + parsed: list[tuple[str, ...]] = [] + for i, entry in enumerate(entries): + source = entry.get("candidates", entry.get("maql")) if isinstance(entry, dict) else entry + parsed.append(tuple(c["maql"] for c in _candidates(source, f"expected_metrics[{i}]"))) + for ref in _PLACEHOLDER_RE.findall(" ".join(parsed[-1])): + if int(ref) >= i: + raise ValueError( + f"expected_metrics[{i}] refers to {{metric/@{ref}}}, which is not an earlier entry" + ) + metrics = tuple(parsed) + + no_deletion = _strings(raw["expect_no_deletion"], "expect_no_deletion") if "expect_no_deletion" in raw else () + executes = _flag(raw, "expect_executes") + suggestions = _flag(raw, "expect_suggestions") + no_overwrite = _flag(raw, "expect_no_overwrite") + if no_create and (candidates or metrics or fields or update or executes or suggestions): + raise ValueError( + "expect_no_create cannot be combined with candidates, expected_metrics, expected_fields, " + "expected_update, expect_executes or expect_suggestions: each needs a written metric" + ) + if (executes or suggestions) and not (candidates or fields): + raise ValueError("expect_executes and expect_suggestions need candidates or expected_fields: a metric to judge") + if update and no_overwrite: + raise ValueError("expected_update writes over an existing metric, which expect_no_overwrite forbids") + if mentions and not no_create: + raise ValueError("expected_mentions needs expect_no_create") + if run_all and not replies: + raise ValueError("run_all_replies needs user_replies") + if metrics and (candidates or not replies): + raise ValueError("expected_metrics needs user_replies and replaces candidates") + if update and candidates: + raise ValueError("expected_update carries its own maql; drop candidates") + + spec = MetricExpectation( + candidates=candidates, + legacy=False, + user_replies=replies, + run_all_replies=run_all, + expect_no_create=no_create, + expected_mentions=mentions, + expect_no_overwrite=no_overwrite, + expect_clarification=_flag(raw, "expect_clarification"), + confirmation_turn=confirmation_turn, + rubric=rubric, + expected_fields=fields, + expected_update=update, + expected_metrics=metrics, + expect_no_deletion=no_deletion, + expect_executes=executes, + score_first_proposal=_flag(raw, "score_first_proposal"), + expect_suggestions=suggestions, + ) + if spec.score_first_proposal and not candidates: + raise ValueError("score_first_proposal needs candidates") + if not ( + candidates + or no_create + or metrics + or update + or fields + or spec.expect_no_overwrite + or spec.expect_clarification + or rubric + or spec.expect_no_deletion + or confirmation_turn is not None + ): + raise ValueError("expected_output checks nothing: give candidates or at least one expect_* key") + return spec + + +def format_matches_kind(fmt: str, kind: str) -> bool: + """Whether a number format string shows the given kind of number. A missing format shows no kind.""" + if not fmt.strip(): + return False + if kind == "percent": + return "%" in fmt + if kind == "currency": + return any(mark in fmt for mark in _CURRENCY_MARKS) + if kind == "integer": + return _DECIMAL_RE.search(fmt) is None + return _DECIMAL_RE.search(fmt) is not None + + +def _norm_text(value: Any) -> str: + return " ".join(str(value or "").split()).casefold() + + +def _tags(value: Any) -> frozenset[str]: + return frozenset(_norm_text(t) for t in (value or [])) + + +def stored_fields(attributes: dict) -> dict: + """The comparable fields of a metric's entity attributes: every writable one, none the server manages.""" + content = attributes.get("content") or {} + return { + "title": attributes.get("title") or "", + "description": attributes.get("description") or "", + "tags": sorted(attributes.get("tags") or []), + "format": content.get("format") or "", + "maql": content.get("maql") or "", + "isHidden": bool(attributes.get("isHidden")), + "isHiddenFromKda": bool(attributes.get("isHiddenFromKda")), + "conditionalFormatting": attributes.get("conditionalFormatting"), + } + + +def field_mismatches(spec: FieldSpec, metric_id: str, attributes: dict | None) -> list[str]: + """How the stored metric differs from the spec; empty when every listed field matches.""" + if attributes is None: + return [f"metric {metric_id!r} is not stored in the workspace"] + stored = stored_fields(attributes) + out: list[str] = [] + if spec.metric_id is not None and spec.metric_id != metric_id: + out.append(f"metric_id: expected {spec.metric_id!r}, got {metric_id!r}") + if spec.title is not None and _norm_text(spec.title) != _norm_text(stored["title"]): + out.append(f"title: expected {spec.title!r}, got {stored['title']!r}") + if spec.format is not None and spec.format.strip() != stored["format"].strip(): + out.append(f"format: expected {spec.format!r}, got {stored['format']!r}") + if spec.format_kind is not None and not format_matches_kind(stored["format"], spec.format_kind): + out.append(f"format: expected a {spec.format_kind} format, got {stored['format']!r}") + out += [f"format: {s!r} not in {stored['format']!r}" for s in spec.format_contains if s not in stored["format"]] + if spec.description is not None and spec.description.strip() != stored["description"].strip(): + out.append(f"description: expected {spec.description!r}, got {stored['description']!r}") + out += [ + f"description: {s!r} not in {stored['description']!r}" + for s in spec.description_contains + if _norm_text(s) not in _norm_text(stored["description"]) + ] + if spec.tags is not None and _tags(spec.tags) != _tags(stored["tags"]): + out.append(f"tags: expected {sorted(spec.tags)}, got {stored['tags']}") + missing = _tags(spec.tags_include) - _tags(stored["tags"]) + if missing: + out.append(f"tags: {sorted(missing)} missing from {stored['tags']}") + return out + + +def update_mismatches(spec: UpdateSpec, before: dict | None, after: dict | None) -> list[str]: + """How an update differs from the spec: a listed field did not change as asked, or another field moved.""" + if before is None: + return [f"metric {spec.metric_id!r} did not exist before the run"] + out = field_mismatches(spec.fields, spec.metric_id, after) + if after is None: + return out + old, new = stored_fields(before), stored_fields(after) + if spec.maql and not any(normalize_maql(m) == normalize_maql(new["maql"]) for m in spec.maql): + out.append(f"maql: expected one of {list(spec.maql)}, got {new['maql']!r}") + listed = { + "title": spec.fields.title is not None, + "description": spec.fields.description is not None or bool(spec.fields.description_contains), + "tags": spec.fields.tags is not None or bool(spec.fields.tags_include), + "format": spec.fields.format is not None + or spec.fields.format_kind is not None + or bool(spec.fields.format_contains), + "maql": bool(spec.maql), + } + for key in old: + same = normalize_maql(old[key]) == normalize_maql(new[key]) if key == "maql" else old[key] == new[key] + if listed.get(key) and same: + # The final value matching is not enough: a request already met before the run proves no update. + out.append(f"{key} did not change although the request asked for it: still {new[key]!r}") + elif not listed.get(key) and not same: + out.append(f"{key} changed although the request did not ask for it: {old[key]!r} -> {new[key]!r}") + return out + + +def substitute_placeholders(maql: str, created_ids: list[str | None]) -> str: + """Replace ``{metric/@n}`` with the id of the metric matched for entry n.""" + + def _sub(match: re.Match) -> str: + index = int(match.group(1)) + metric_id = created_ids[index] if index < len(created_ids) else None + return f"{{metric/{metric_id}}}" if metric_id else match.group(0) + + return _PLACEHOLDER_RE.sub(_sub, maql) + + +def first_proposed_maql(texts: list[str]) -> str | None: + """The MAQL of the first formula the agent showed in a code span or a fenced block, if any.""" + for text in texts: + found = _PROPOSED_MAQL_RE.search(text or "") + if found: + return (found.group(1) or found.group(2)).strip() + return None + + +def missing_mentions(mentions: tuple[str, ...], texts: list[str]) -> list[str]: + """The expected mentions no agent reply names, compared case-insensitively.""" + joined = _norm_text(" ".join(texts)) + return [m for m in mentions if _norm_text(m) not in joined] diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_metric_workspace.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_metric_workspace.py new file mode 100644 index 000000000..41bbc8773 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_metric_workspace.py @@ -0,0 +1,233 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Workspace access the metric-skill evaluator needs beyond the chat: the metrics, a test execution, the conversation +history. + +Metrics are read through the SDK's entity API. Restores go through the metric entity endpoint, never through the +agent's ``create_metric`` tool: that tool writes its defaults over any field it is not given, so a restore through it +would change the metric it tries to restore. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable +from typing import Any + +import httpx +from gooddata_api_client.exceptions import ApiException, NotFoundException +from gooddata_sdk import GoodDataSdk +from gooddata_sdk.utils import load_all_entities + +from gooddata_eval.core._retry import RETRYABLE_STATUS_CODES, is_retryable_request_error, retry_transient +from gooddata_eval.core.agentic._clarification import _page_with_retry + +_METRICS = "/api/v1/entities/workspaces/{ws}/metrics" +_EXECUTE = "/api/v1/actions/workspaces/{ws}/execution/afm/execute" +_ITEMS = "/api/v1/ai/workspaces/{ws}/chat/conversations/{conversation_id}/items" +_JSON_API = "application/vnd.gooddata.api+json" +# Every attribute a metric write accepts except areRelationsValid, which the server computes. +_WRITABLE = ("title", "description", "tags", "content", "isHidden", "isHiddenFromKda", "conditionalFormatting") + + +class WorkspaceReadError(RuntimeError): + """The harness could not read the workspace or the conversation, so the run cannot be scored.""" + + +class _TransientStatus(Exception): + """A request answered with a retryable status; carries the response for the final report.""" + + def __init__(self, response: httpx.Response): + super().__init__(f"HTTP {response.status_code}") + self.response = response + + +def _is_transient_request(exc: Exception) -> bool: + return isinstance(exc, _TransientStatus) or is_retryable_request_error(exc) + + +class MetricWorkspace: + """Reads and restores one workspace's metrics, executes a metric, and reads a conversation's items.""" + + def __init__(self, host: str, token: str, workspace_id: str, sdk: GoodDataSdk, *, timeout: float = 60.0): + self._base = host.rstrip("/") + self._ws = workspace_id + self._entities = sdk._client.entities_api + self._auth = {"Authorization": f"Bearer {token}"} + self._client = httpx.Client(timeout=timeout) + # Metrics a run could not put back; while any is listed, later runs are not driven. + self.unrestored: list[str] = [] + + def close(self) -> None: + self._client.close() + + def list_metrics(self) -> dict[str, dict]: + """Every metric of the workspace, as metric id -> entity attributes. + + Raises: + WorkspaceReadError: A page could not be read after the transient retries. + """ + list_page = functools.partial(self._entities.get_all_entities_metrics, self._ws, _check_return_type=False) + try: + data = load_all_entities(functools.partial(_page_with_retry, list_page)).data + except (ApiException, httpx.HTTPError, OSError) as exc: + raise WorkspaceReadError(f"could not list metrics: {exc}") from exc + out: dict[str, dict] = {} + for obj in data: + # A row the run could not restore from makes the snapshot useless, so it fails the snapshot. + attributes = obj.get("attributes") if isinstance(obj, dict) else None + if not isinstance(obj, dict) or not obj.get("id") or not isinstance(attributes, dict): + raise WorkspaceReadError(f"could not list metrics: unreadable row {obj!r}") + if not isinstance(attributes.get("content"), dict) or not attributes["content"].get("maql"): + raise WorkspaceReadError(f"could not list metrics: metric {obj['id']!r} has no MAQL") + out[obj["id"]] = attributes + return out + + def get_metric(self, metric_id: str) -> dict | None: + """One metric's entity attributes, or None when it does not exist. + + Raises: + WorkspaceReadError: The lookup failed for a reason other than a 404. + """ + try: + doc = self._entities.get_entity_metrics(self._ws, metric_id, _check_return_type=False) + except NotFoundException: + return None + except (ApiException, httpx.HTTPError, OSError) as exc: + raise WorkspaceReadError(f"could not read metric {metric_id}: {exc}") from exc + # The document is a model even with _check_return_type=False; only its data is a plain dict. + data = doc.get("data") if isinstance(doc, dict) else getattr(doc, "data", None) + attributes = data.get("attributes") if isinstance(data, dict) else None + if not isinstance(attributes, dict): + raise WorkspaceReadError(f"could not read metric {metric_id}: no attributes in {data!r}") + return attributes + + def restore_metric(self, metric_id: str, attributes: dict, *, exists: bool) -> bool: + """Write a metric back to the given attributes, retrying transient failures; True on success. + + A failure is printed, not raised: the caller reports it as a harness error. + """ + body = { + "data": { + "id": metric_id, + "type": "metric", + "attributes": {k: attributes[k] for k in _WRITABLE if attributes.get(k) is not None}, + } + } + base = f"{self._base}{_METRICS.format(ws=self._ws)}" + headers = {**self._auth, "Content-Type": _JSON_API} + + def _send() -> httpx.Response: + if exists: + resp = self._client.put(f"{base}/{metric_id}", json=body, headers=headers) + else: + resp = self._client.post(base, json=body, headers=headers) + if resp.status_code in RETRYABLE_STATUS_CODES: + raise _TransientStatus(resp) + return resp + + try: + resp = retry_transient(_send, is_retryable=_is_transient_request) + except _TransientStatus as exc: + resp = exc.response + except httpx.HTTPError as exc: + print(f"[CLEANUP] Could not restore metric {metric_id}: {exc}") + return False + if not resp.is_success: + print(f"[CLEANUP] Could not restore metric {metric_id}: HTTP {resp.status_code} {resp.text[:200]}") + return resp.is_success + + def execute_metric(self, metric_id: str) -> str | None: + """Compute the metric alone; None when it computes, else the reason the platform gave. + + Only a 4xx rejection is a reason. A rate limit (429) or a service fault (5xx) is retried, and one that + outlasts the retries is raised: it says nothing about the metric. + + Raises: + WorkspaceReadError: The request did not reach the platform, the platform did not serve it, or its + answer was unreadable. + """ + afm = { + "execution": { + "measures": [ + { + "localIdentifier": "m", + "definition": {"measure": {"item": {"identifier": {"id": metric_id, "type": "metric"}}}}, + } + ], + "attributes": [], + "filters": [], + }, + "resultSpec": {"dimensions": [{"localIdentifier": "d", "itemIdentifiers": ["measureGroup"]}]}, + } + url = f"{self._base}{_EXECUTE.format(ws=self._ws)}" + + def _served(step: str, send: Callable[[], httpx.Response]) -> httpx.Response: + """The response once it is no rate limit or service fault, retrying those; raises when they persist.""" + + def _attempt() -> httpx.Response: + resp = send() + if resp.status_code in RETRYABLE_STATUS_CODES: + raise _TransientStatus(resp) + return resp + + try: + resp = retry_transient(_attempt, is_retryable=_is_transient_request) + except _TransientStatus as exc: + resp = exc.response + if resp.status_code in RETRYABLE_STATUS_CODES or resp.status_code >= 500: + raise WorkspaceReadError(f"could not execute metric {metric_id}: {step} HTTP {resp.status_code}") + return resp + + try: + resp = _served( + "execute", + lambda: self._client.post(url, json=afm, headers={**self._auth, "Content-Type": "application/json"}), + ) + if not resp.is_success: + return f"execute: HTTP {resp.status_code} {resp.text[:300]}" + body = resp.json() + response = body.get("executionResponse") if isinstance(body, dict) else None + links = response.get("links") if isinstance(response, dict) else None + result_id = links.get("executionResult") if isinstance(links, dict) else None + if not isinstance(result_id, str) or not result_id: + raise WorkspaceReadError(f"could not execute metric {metric_id}: no executionResult in {body!r}"[:400]) + result = _served("result", lambda: self._client.get(f"{url}/result/{result_id}", headers=self._auth)) + if not result.is_success: + return f"result: HTTP {result.status_code} {result.text[:300]}" + computed = result.json() + except (httpx.HTTPError, ValueError) as exc: + raise WorkspaceReadError(f"could not execute metric {metric_id}: {type(exc).__name__}: {exc}") from exc + if not isinstance(computed, dict) or not isinstance(computed.get("data"), list): + # A computed result carries its data list, empty or not; a 200 without one proves no computation. + raise WorkspaceReadError(f"could not execute metric {metric_id}: no data list in {str(computed)[:200]}") + return None + + def conversation_items(self, conversation_id: str) -> list[dict[str, Any]]: + """The conversation's stored items, oldest first. + + Raises: + WorkspaceReadError: The items could not be read. + """ + try: + resp = self._client.get( + f"{self._base}{_ITEMS.format(ws=self._ws, conversation_id=conversation_id)}", headers=self._auth + ) + resp.raise_for_status() + body = resp.json() + except (httpx.HTTPError, ValueError) as exc: + raise WorkspaceReadError(f"could not read conversation {conversation_id}: {exc}") from exc + key = next((k for k in ("items", "data") if isinstance(body, dict) and k in body), None) + items = body[key] if key is not None else None + if not isinstance(items, list): + # A shape without its items list would read as a conversation with no suggestions, and pass. + raise WorkspaceReadError(f"conversation {conversation_id}: no items list in {str(body)[:200]}") + return items + + +def last_suggestions(items: list[dict[str, Any]]) -> dict | None: + """The suggestions of the latest assistant item that carries them.""" + for item in reversed(items): + content = item.get("content") if isinstance(item, dict) else None + if isinstance(content, dict) and isinstance(content.get("suggestions"), dict): + return content["suggestions"] + return None diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 17d19182e..22de78da5 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -9,9 +9,12 @@ from dataclasses import dataclass, field from typing import Any +import httpx from gooddata_api_client.exceptions import NotFoundException from gooddata_sdk import GoodDataSdk +from gooddata_eval.core.agentic._clarification import read_first_reply +from gooddata_eval.core.agentic._conversation_context import asks_in_text from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._failure_message import FailedCheck, failed_check_records from gooddata_eval.core.agentic._gate import ( @@ -21,7 +24,18 @@ log_gate_scores, stamp_gate_metadata, ) +from gooddata_eval.core.agentic._metric_expectation import ( + MetricExpectation, + field_mismatches, + first_proposed_maql, + missing_mentions, + parse_expectation, + substitute_placeholders, + update_mismatches, +) +from gooddata_eval.core.agentic._metric_workspace import MetricWorkspace, WorkspaceReadError, last_suggestions from gooddata_eval.core.agentic._outcome import ( + CHAT_FAILURE_EXITS, agentic_detail, agentic_success, chat_failure_exit, @@ -42,11 +56,13 @@ from gooddata_eval.core.chat.render import render_answer_text from gooddata_eval.core.chat.sse_client import ChatClient, ChatError from gooddata_eval.core.config import ReasoningEffort +from gooddata_eval.core.evaluators._llm_judge import JudgeVerdict, LLMJudge, score_run from gooddata_eval.core.evaluators._maql import normalize_maql from gooddata_eval.core.langfuse.item_scope import item_scope from gooddata_eval.core.models import ( AgenticAssertionError, AgenticEvalOutcome, + ChatResult, LoopExit, ReasoningStepEvent, ToolCallEvent, @@ -60,12 +76,49 @@ except ImportError: _OpenAI: Any = None + +def _judge_provider_errors() -> tuple[type[BaseException], ...]: + """The failures of a judge request at the provider (timeout, outage, rate limit): they leave a run unscored.""" + transport: tuple[type[BaseException], ...] = (httpx.HTTPError, TimeoutError, ConnectionError) + try: + from openai import OpenAIError # noqa: PLC0415 + except ImportError: + return transport + return (OpenAIError, *transport) + + _DEFAULT_K = 1 _DEFAULT_MAX_ITERATIONS = 7 # Seconds to wait before each check that a deleted metric stayed deleted, so the checks fall # 3, 6 and 12 seconds after the delete; see _delete_metric. _DELETE_RECHECK_DELAYS_S: tuple[float, ...] = (3.0, 3.0, 6.0) +# Every check score an options-shape item can publish, beside metric_created and maql_correct. Each is published only +# for an item that opts into it; metric_scored only when the harness could not score a run, which is then labelled +# ERROR. metric_first_proposal_correct is a diagnostic and never gates. +METRIC_CONDITIONAL_SCORES = ( + "metric_not_created", + "metric_reuse_named", + "metric_no_overwrite", + "metric_clarified", + "metric_confirmed_first", + "metric_rubric", + "metric_fields_correct", + "metric_updated", + "metric_all_created", + "metric_none_deleted", + "metric_executes", + "metric_suggestions", + "metric_scored", +) +METRIC_DIAGNOSTIC_SCORES = ("metric_first_proposal_correct",) + +_RUBRIC_STEPS = [ + "EXPECTED OUTPUT states the criteria the assistant's behaviour must meet.", + "ACTUAL OUTPUT holds every assistant reply of the conversation, in order.", + "Pass only when every reply meets every criterion. Fail when any reply breaks one.", +] + def _best_maql_match(actual_maql: str, expected_outputs: list[dict]) -> tuple[bool, str]: """Try actual MAQL against every candidate; return (matched, best_expected_maql). @@ -192,6 +245,30 @@ class MetricRunResult: timings: PhaseTimings = field(default_factory=PhaseTimings) # This run's own wall time, from just before its first chat call to the end of its scoring. run_latency_s: float = 0.0 + # The gating checks of an options-shape item, by score name; empty for the original shape. + checks: dict[str, bool] = field(default_factory=dict) + diagnostics: dict[str, bool] = field(default_factory=dict) + # Why each failed check failed, by score name. + check_details: dict[str, Any] = field(default_factory=dict) + # Set when the harness could not score the run (workspace unreadable, judge failed): the run is labelled + # ERROR rather than blamed on the agent. + harness_error: str | None = None + + @property + def passed(self) -> bool: + """Every gating check passed and the harness scored the run fully. + + The original shape gates on the created metric and its MAQL. A harness error (a check the harness could not + read, a restore that failed) makes the run unscored: it never passes, and its checks stay for diagnosis. + An options-shape run whose chat failed does not pass either: the loop sends a turn only while the item + still needs one (a scripted reply, a missing metric), so checks that held on the cut-off conversation prove + nothing. Its label is the FAIL of that exit. + """ + if self.harness_error: + return False + if self.checks: + return self.exit_reason not in CHAT_FAILURE_EXITS and all(self.checks.values()) + return self.metric_created and self.maql_correct @dataclass @@ -211,17 +288,8 @@ def _extract_metric_result(tool_call_events: list[ToolCallEvent]) -> dict | None after a validation error, the earlier failed attempt must not shadow it. Shared with ``conversation.py``, which imports this instead of keeping its own copy. """ - for tc in reversed(tool_call_events): - if tc.function_name != "create_metric" or not tc.result: - continue - result_data = tc.parsed_result() - if not isinstance(result_data, dict): - continue - payload = result_data.get("data", result_data) - if not isinstance(payload, dict) or not payload or payload.get("isError"): - continue - return payload - return None + writes = _successful_writes(tool_call_events) + return writes[-1] if writes else None def _extract_created_metric_ids(tool_call_events: list[ToolCallEvent]) -> list[str]: @@ -236,7 +304,8 @@ def _extract_created_metric_ids(tool_call_events: list[ToolCallEvent]) -> list[s if tc.function_name != "create_metric" or not tc.result: continue result_data = tc.parsed_result() - if not result_data: + if not isinstance(result_data, dict): + # A result that is not an object names no metric; it must not stop the ids of the other calls. continue data = result_data.get("data", result_data) if not isinstance(data, dict) or data.get("created_new") is False: @@ -247,6 +316,73 @@ def _extract_created_metric_ids(tool_call_events: list[ToolCallEvent]) -> list[s return metric_ids +def _successful_writes(tool_call_events: list[ToolCallEvent]) -> list[dict]: + """The payload of every successful ``create_metric`` call, in call order; updates included.""" + out: list[dict] = [] + for tc in tool_call_events: + if tc.function_name != "create_metric" or not tc.result: + continue + result_data = tc.parsed_result() + if not isinstance(result_data, dict): + continue + payload = result_data.get("data", result_data) + if isinstance(payload, dict) and payload and not payload.get("isError"): + out.append(payload) + return out + + +class MetricMatchUndecided(RuntimeError): + """The assignment search ran out of steps before it could tell whether every expected metric was written.""" + + +# Steps of the assignment search. A real item ends in a few dozen; only many written metrics sharing one formula +# come near, and their factorial count of assignments must not stall the item. +_MATCH_SEARCH_STEPS = 10_000 + + +def _match_expected_metrics(expected: tuple[tuple[str, ...], ...], writes: list[dict]) -> list[str | None]: + """For each expected metric in order, the id of a written metric whose MAQL matches it, or None. + + Each id is judged by its latest write, since a rewrite replaces the earlier formula, and each id fills one + entry. Every assignment is tried, so a candidate that two entries both accept cannot starve the other; the + first complete assignment wins, else the one that matches the most entries. + + Raises: + MetricMatchUndecided: The search used ``_MATCH_SEARCH_STEPS`` steps without a complete assignment, so + whether one exists is unknown. + """ + latest: dict[str, str] = {} + for write in writes: + if write.get("metric_id"): + latest[write["metric_id"]] = normalize_maql(write.get("maql") or "") + best: list[str | None] = [None] * len(expected) + steps = 0 + + def _search(assigned: list[str | None]) -> bool: + nonlocal best, steps + steps += 1 + if steps > _MATCH_SEARCH_STEPS: + raise MetricMatchUndecided( + f"no verdict after {_MATCH_SEARCH_STEPS} assignments of {len(latest)} written metrics " + f"to {len(expected)} expected ones" + ) + if len(assigned) == len(expected): + if sum(a is not None for a in assigned) > sum(b is not None for b in best): + best = list(assigned) + return None not in assigned + candidates = expected[len(assigned)] + for metric_id, maql in latest.items(): + matches = metric_id not in assigned and any( + maql == normalize_maql(substitute_placeholders(c, assigned)) for c in candidates + ) + if matches and _search([*assigned, metric_id]): + return True + return _search([*assigned, None]) + + _search([]) + return best + + def _delete_metric(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> None: """Delete a metric created during evaluation. @@ -304,6 +440,206 @@ def _metric_exists(sdk: GoodDataSdk, workspace_id: str, metric_id: str) -> bool: return True +def _stop_after_write(spec: MetricExpectation, replies_left: int, writes: list[dict]) -> bool: + """Whether a turn that wrote a metric ends the run. + + The original shape stops at the first metric, as before. An item that scripts further replies after a write + (``run_all_replies``), or expects several metrics, keeps going while replies are left and metrics are missing. + """ + if spec.legacy or spec.expect_no_create: + return True + if spec.run_all_replies and replies_left: + return False + if spec.expected_metrics: + try: + return None not in _match_expected_metrics(spec.expected_metrics, writes) or replies_left == 0 + except MetricMatchUndecided: + return True # No later turn can make the match decidable; the score reports it. + return True + + +def _judge_rubric(judge: LLMJudge, rubric: str, transcript: list[tuple[str, str]]) -> JudgeVerdict: + """Grade the whole conversation, user and assistant turns in order, against the rubric.""" + conversation = "\n\n".join(f"{role}: {text}" for role, text in transcript) + first_user = next((text for role, text in transcript if role == "User"), "") + return score_run(judge, input=first_user, expected_output=rubric, actual_output=conversation) + + +def _score_options_run( + spec: MetricExpectation, + workspace: MetricWorkspace, + *, + conversation_id: str, + transcript: list[tuple[str, str]], + texts: list[str], + first_turn: ChatResult | None, + first_write_call_turn: int | None, + writes: list[dict], + metric_result: dict | None, + before: dict[str, dict], + judge: LLMJudge | None, + answered: bool = True, +) -> tuple[dict[str, bool], dict[str, bool], dict[str, Any], str | None]: + """The gating checks, diagnostics, failure reasons and harness error of one options-shape run. + + A check the harness could not evaluate (the workspace or the conversation was unreadable, the judge gave no + verdict) reads False and sets the harness error, so the run is labelled ERROR instead of blamed on the agent. + """ + checks: dict[str, bool] = {} + diagnostics: dict[str, bool] = {} + details: dict[str, Any] = {} + harness_errors: list[str] = [] + + def _check(name: str, ok: bool, why: Any = None) -> None: + checks[name] = ok + if not ok and why: + details[name] = why + + def _harness(name: str, reason: str) -> None: + _check(name, False, f"harness: {reason}") + harness_errors.append(f"{name}: {reason}") + + target_id = (metric_result or {}).get("metric_id") + if spec.candidates: + actual = (metric_result or {}).get("maql", "") + _check("metric_created", metric_result is not None) + _check( + "maql_correct", + metric_result is not None and _best_maql_match(actual, spec.candidates)[0], + {"expected": [c["maql"] for c in spec.candidates], "actual": actual}, + ) + if spec.expect_no_create: + # A run whose chat failed before any complete answer proves no refusal. + _check( + "metric_not_created", + answered and not writes, + {"written": [w.get("metric_id") for w in writes]} if answered else "the agent never completed an answer", + ) + if spec.expected_mentions: + missing = missing_mentions(spec.expected_mentions, texts) + _check("metric_reuse_named", not missing, {"missing": missing}) + if spec.expect_no_overwrite: + # A metric is overwritten only when it existed before the run; editing one the run created is not. + overwritten = list(dict.fromkeys(w.get("metric_id") for w in writes if w.get("metric_id") in before)) + _check("metric_no_overwrite", not overwritten, {"overwritten": overwritten}) + if spec.expect_clarification: + asked = ( + first_turn is not None + and read_first_reply(first_turn, asked_in_text=asks_in_text(first_turn, texts[0] if texts else "")).asked + ) + _check( + "metric_clarified", + asked and first_write_call_turn != 1, + {"asked": asked, "write_turn": first_write_call_turn}, + ) + if spec.confirmation_turn is not None: + ok = first_write_call_turn is None or first_write_call_turn >= spec.confirmation_turn + _check( + "metric_confirmed_first", + ok, + {"first_create_metric_turn": first_write_call_turn, "allowed_from": spec.confirmation_turn}, + ) + if spec.rubric and judge is not None: + try: + verdict = _judge_rubric(judge, spec.rubric, transcript) + except _judge_provider_errors() as exc: + verdict = JudgeVerdict(passed=False, reasoning="", error=f"{type(exc).__name__}: {exc}") + if verdict.error is not None: + _harness("metric_rubric", f"judge gave no verdict: {verdict.error}") + else: + _check("metric_rubric", verdict.passed, verdict.reasoning) + try: + if spec.expected_fields is not None: + # The metric the agent wrote, never the id the item names: a pre-existing metric of that id could pass. + if target_id is None: + _check("metric_fields_correct", False, "no metric was written") + else: + mismatches = field_mismatches(spec.expected_fields, target_id, workspace.get_metric(target_id)) + _check("metric_fields_correct", not mismatches, mismatches) + if spec.expected_update is not None: + update_id = spec.expected_update.metric_id + mismatches = update_mismatches(spec.expected_update, before.get(update_id), workspace.get_metric(update_id)) + _check("metric_updated", not mismatches, mismatches) + if spec.expect_no_deletion: + gone = [m for m in spec.expect_no_deletion if workspace.get_metric(m) is None] + _check("metric_none_deleted", not gone, {"missing": gone}) + if spec.expect_executes: + reason = workspace.execute_metric(target_id) if target_id else "no metric was written" + _check("metric_executes", reason is None, reason) + if spec.expect_suggestions: + # QA accepts a turn without suggestions; when there are some, the first must name the new metric. + suggestions = last_suggestions(workspace.conversation_items(conversation_id)) + actions = (suggestions or {}).get("actions") + if actions is None: + actions = [] # Only an absent list means no actions; any other value is checked below. + if not isinstance(actions, list) or not all(isinstance(a, dict) for a in actions): + raise WorkspaceReadError(f"conversation {conversation_id}: unreadable suggestion actions {actions!r}") + if not actions or target_id is None: + _check("metric_suggestions", True) + else: + first = f"{actions[0].get('query', '')} {actions[0].get('label', '')}".casefold() + title = str((metric_result or {}).get("title") or "").casefold() + named = (bool(title) and title in first) or f"{{metric/{target_id}}}".casefold() in first + _check("metric_suggestions", named, {"first_action": actions[0], "metric": target_id}) + except WorkspaceReadError as exc: + _harness("metric_scored", str(exc)) + if spec.expected_metrics: + try: + matched = _match_expected_metrics(spec.expected_metrics, writes) + except MetricMatchUndecided as exc: + _harness("metric_all_created", str(exc)) + else: + _check( + "metric_all_created", + None not in matched, + {"matched": matched, "written": [(w.get("metric_id"), w.get("maql")) for w in writes]}, + ) + if spec.score_first_proposal: + # An agent that wrote without showing a formula first proposed the MAQL of its first write, not its last. + proposed = first_proposed_maql(texts) or (writes[0].get("maql") if writes else None) + diagnostics["metric_first_proposal_correct"] = ( + bool(proposed) and _best_maql_match(proposed or "", spec.candidates)[0] + ) + if not diagnostics["metric_first_proposal_correct"]: + details["metric_first_proposal_correct"] = {"proposed": proposed} + return checks, diagnostics, details, ("; ".join(harness_errors) or None) + + +def _restore_workspace( + workspace: MetricWorkspace, before: dict[str, dict], writes: list[dict], guarded: tuple[str, ...] +) -> list[str]: + """Put back every pre-existing metric the run wrote over, and every guarded metric it removed. + + Returns the ids it could not put back. Guarded metrics go first: one the run wrote over and then removed is + recreated with its original attributes, so the overwrite restore, which updates a metric that exists, skips it. + """ + failed: list[str] = [] + recreated: set[str] = set() + for metric_id in guarded: + if metric_id not in before: + continue + try: + gone = workspace.get_metric(metric_id) is None + except WorkspaceReadError as exc: + print(f"[CLEANUP] Could not check metric {metric_id}: {exc}") + failed.append(metric_id) + continue + if not gone: + continue + if workspace.restore_metric(metric_id, before[metric_id], exists=False): + recreated.add(metric_id) + else: + failed.append(metric_id) + failed.extend( + metric_id + for metric_id in dict.fromkeys(w.get("metric_id") for w in writes) + if metric_id in before + and metric_id not in recreated + and not workspace.restore_metric(metric_id, before[metric_id], exists=True) + ) + return list(dict.fromkeys(failed)) + + def _execute_single_metric_run( client: ChatClient, sdk: GoodDataSdk, @@ -312,13 +648,41 @@ def _execute_single_metric_run( question: str, expected_outputs: list[dict], max_iterations: int, + *, + spec: MetricExpectation | None = None, + workspace: MetricWorkspace | None = None, + judge: LLMJudge | None = None, ) -> MetricRunResult: """Drive one full multi-turn metric-skill conversation and evaluate the result. Any metric the agent creates during this run is deleted on the way out (see ``_delete_metric``) so it cannot leak into — and be reused by — a later test - sharing the workspace. + sharing the workspace. An options-shape item also gets back every metric the run + wrote over, from a snapshot taken before its first turn; without that snapshot the + agent is not driven at all, since nothing it overwrote could be put back. """ + spec = spec or MetricExpectation(candidates=expected_outputs) + before: dict[str, dict] = {} + if workspace is not None: + try: + if workspace.unrestored: + # A snapshot now would record an earlier run's leftover as the original. + raise WorkspaceReadError(f"an earlier run could not put back {workspace.unrestored}") + before = workspace.list_metrics() + except WorkspaceReadError as exc: + print(f"[SNAPSHOT] Could not read the workspace metrics before the run: {exc}") + return MetricRunResult( + conversation_id=conversation_id, + metric_result=None, + metric_created=False, + actual_maql="", + maql_correct=False, + total_turns=0, + exit_reason=LoopExit.NOT_RUN, + checks={"metric_scored": False}, + check_details={"metric_scored": f"harness: snapshot: {exc}"}, + harness_error=f"snapshot: {exc}", + ) metric_result: dict | None = None created_metric_ids: list[str] = [] turns = 0 @@ -332,6 +696,27 @@ def _execute_single_metric_run( turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time tool_index_offset = 0 reasoning_index_offset = 0 + replies = list(spec.user_replies) + transcript: list[tuple[str, str]] = [("User", question)] + texts: list[str] = [] + first_turn: ChatResult | None = None + first_write_call_turn: int | None = None + writes: list[dict] = [] + answered_turns = 0 + result: MetricRunResult | None = None + + def _record_turn(result: ChatResult, events: list[ToolCallEvent]) -> str: + """Note the turn's writes and text for the options-shape checks; returns the rendered text.""" + nonlocal first_turn, first_write_call_turn + writes.extend(_successful_writes(events)) + if first_write_call_turn is None and any(tc.function_name == "create_metric" for tc in events): + first_write_call_turn = turns + if first_turn is None: + first_turn = result + text = render_answer_text(result) + texts.append(text) + transcript.append(("Assistant", text)) + return text try: # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop that @@ -360,10 +745,12 @@ def _execute_single_metric_run( for metric_id in _extract_created_metric_ids(partial.tool_call_events or []): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) + _record_turn(partial, partial.tool_call_events or []) exit_reason = chat_failure_exit(exc) break agent_elapsed = time.monotonic() - agent_started timings.agent_s += agent_elapsed + answered_turns += 1 reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( @@ -372,34 +759,47 @@ def _execute_single_metric_run( tool_index_offset=tool_index_offset, reasoning_index_offset=reasoning_index_offset, ) - all_tool_call_events.extend(chat_result.tool_call_events or []) + turn_events = chat_result.tool_call_events or [] + all_tool_call_events.extend(turn_events) all_reasoning_step_events.extend(chat_result.reasoning_step_events or []) steps += chat_result.reasoning_step_count - for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []): + for metric_id in _extract_created_metric_ids(turn_events): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) - candidate = _extract_metric_result(chat_result.tool_call_events or []) + response_text = _record_turn(chat_result, turn_events) + candidate = _extract_metric_result(turn_events) if candidate is not None: - log_timer( - f"[timer] metric_skill {conversation_id} GoodData turn {turns} complete after " - f"{agent_elapsed:.2f}s; metric result received" - ) metric_result = candidate - exit_reason = LoopExit.SUCCESS - break - response_text = render_answer_text(chat_result) - if not response_text and not chat_result.tool_call_events: + if _stop_after_write(spec, len(replies), writes): + log_timer( + f"[timer] metric_skill {conversation_id} GoodData turn {turns} complete after " + f"{agent_elapsed:.2f}s; metric result received" + ) + exit_reason = LoopExit.SUCCESS + break + if not response_text and not turn_events: exit_reason = LoopExit.AGENT_SILENT break if _iteration >= max_iterations - 1: break + if replies: + current_question = replies.pop(0) + transcript.append(("User", current_question)) + continue + if not spec.legacy and ( + not spec.candidates or spec.expect_no_create or spec.run_all_replies or spec.expected_metrics + ): + # The scripted replies are spent and the item hands the rest to no one: it has no MAQL for the + # simulated user, or it scripts the whole conversation (run_all_replies, several metrics). + exit_reason = LoopExit.STOPPED_EARLY + break log_timer( f"[timer] metric_skill {conversation_id} GoodData turn {turns} complete after " f"{agent_elapsed:.2f}s; waiting for gpt-4o-mini simulated user" ) simulated_started = time.monotonic() try: - current_question = generate_simulated_response(response_text, expected_outputs, question) + current_question = generate_simulated_response(response_text, spec.candidates, question) except SimulatedResponseError as exc: simulated_elapsed = time.monotonic() - simulated_started timings.simulated_user_s += simulated_elapsed @@ -414,6 +814,7 @@ def _execute_single_metric_run( break simulated_elapsed = time.monotonic() - simulated_started timings.simulated_user_s += simulated_elapsed + transcript.append(("User", current_question)) log_timer( f"[timer] metric_skill {conversation_id} gpt-4o-mini simulated user complete after " f"{simulated_elapsed:.2f}s" @@ -421,8 +822,27 @@ def _execute_single_metric_run( actual_maql = (metric_result or {}).get("maql", "") metric_created = metric_result is not None - maql_correct, _ = _best_maql_match(actual_maql, expected_outputs) if metric_created else (False, "") - return MetricRunResult( + maql_correct, _ = _best_maql_match(actual_maql, spec.candidates) if metric_created else (False, "") + checks: dict[str, bool] = {} + diagnostics: dict[str, bool] = {} + check_details: dict[str, Any] = {} + harness_error: str | None = None + if workspace is not None: + checks, diagnostics, check_details, harness_error = _score_options_run( + spec, + workspace, + conversation_id=conversation_id, + transcript=transcript, + texts=texts, + first_turn=first_turn, + first_write_call_turn=first_write_call_turn, + writes=writes, + metric_result=metric_result, + before=before, + judge=judge, + answered=answered_turns > 0, + ) + result = MetricRunResult( conversation_id=conversation_id, metric_result=metric_result, metric_created=metric_created, @@ -438,10 +858,27 @@ def _execute_single_metric_run( exit_reason=exit_reason, turns_used=turns, run_latency_s=time.monotonic() - run_started, + checks=checks, + diagnostics=diagnostics, + check_details=check_details, + harness_error=harness_error, ) + return result finally: for metric_id in created_metric_ids: _delete_metric(sdk, workspace_id, metric_id) + if workspace is not None: + unrestored = _restore_workspace(workspace, before, writes, spec.expect_no_deletion) + if unrestored: + # Later runs of the item are skipped rather than scored against a changed workspace. + workspace.unrestored.extend(unrestored) + if result is not None: + # Unscored like a check the harness could not read: metric_scored reads False with the reason. + note = f"restore failed for {unrestored}" + result.harness_error = f"{result.harness_error}; {note}" if result.harness_error else note + result.checks["metric_scored"] = False + prior = result.check_details.get("metric_scored") + result.check_details["metric_scored"] = f"{prior}; harness: {note}" if prior else f"harness: {note}" def run_agentic_metric_skill( @@ -459,10 +896,20 @@ def run_agentic_metric_skill( ) -> AgenticMetricSummary: """Run the metric-skill agentic evaluation K times and return a summary. - ``expected_output`` may be a single candidate dict or a list of candidate dicts. - The run passes when the actual MAQL matches any candidate after normalisation. + ``expected_output`` is a single candidate dict, a list of candidate dicts, or the options shape of + ``_metric_expectation``. With candidates, a run passes when the actual MAQL matches any of them + after normalisation; the options shape adds the checks it names. + + Raises: + ValueError: The options shape is malformed. Raised before any request. + ImportError, OSError: The item has a rubric and the LLM judge cannot be built (the llm-judge extra + or OPENAI_API_KEY is missing). Raised before any request. """ - expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] + spec = parse_expectation(expected_output) + expected_outputs = spec.candidates + # Built before the first request, as general_question does: a missing judge is a setup error to + # report at once, not something to find after every run's conversation. + judge = LLMJudge(_RUBRIC_STEPS) if spec.rubric else None run_results: list[MetricRunResult] = [] client = ChatClient( host=host, @@ -473,15 +920,26 @@ def run_agentic_metric_skill( user_context=user_context, ) sdk = GoodDataSdk.create(host, token) + workspace = MetricWorkspace(host, token, workspace_id, sdk) if spec.needs_workspace else None + + def _run(conversation_id: str) -> MetricRunResult: + return _execute_single_metric_run( + client, + sdk, + workspace_id, + conversation_id, + question, + expected_outputs, + max_iterations, + spec=spec, + workspace=workspace, + judge=judge, + ) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - run_results.append( - _execute_single_metric_run( - client, sdk, workspace_id, conv_id_0, question, expected_outputs, max_iterations - ) - ) + run_results.append(_run(conv_id_0)) finally: if initial_conversation_id is None: # only delete conversations we created client.delete_conversation(conv_id_0) @@ -489,19 +947,21 @@ def run_agentic_metric_skill( for _ in range(1, k): conv_id = client.create_conversation() try: - run_results.append( - _execute_single_metric_run( - client, sdk, workspace_id, conv_id, question, expected_outputs, max_iterations - ) - ) + run_results.append(_run(conv_id)) finally: client.delete_conversation(conv_id) finally: client.close() - - pass_at_k = any(r.metric_created and r.maql_correct for r in run_results) - pass_power_k = all(r.metric_created and r.maql_correct for r in run_results) - best = max(run_results, key=lambda r: (r.maql_correct, r.metric_created)) + if workspace is not None: + workspace.close() + + pass_at_k = any(r.passed for r in run_results) + pass_power_k = all(r.passed for r in run_results) + # A graded run describes the item better than an unscored one, whatever their check counts. + best = max( + run_results, + key=lambda r: (r.passed, r.harness_error is None, sum(r.checks.values()), r.maql_correct, r.metric_created), + ) return AgenticMetricSummary( run_results=run_results, pass_at_k=pass_at_k, @@ -514,6 +974,36 @@ class MetricSkillAssertionError(AgenticAssertionError): """Raised when a metric-skill evaluation fails.""" +def _quality_checks(run: MetricRunResult) -> dict[str, bool]: + """The checks a run's quality score counts. + + An unscored run (``harness_error``) counts as ``metric_scored`` False alone: its other checks are observations + the harness could not complete, and an ERROR run must not report the quality of a graded one. + """ + if run.harness_error: + return {"metric_scored": False} + return run.checks or {"metric_created": run.metric_created, "maql_correct": run.maql_correct} + + +def _verdict_fields(run: MetricRunResult) -> dict[str, Any]: + """A run's checks as top-level booleans of its detail, which the report's quality score counts. + + The original shape keeps ``metric_created`` and ``maql_correct``. An options-shape item puts its own checks there + instead, so an item without candidates is not scored on a MAQL it never asked for; the failure reasons and the + diagnostics sit beside them as dicts, which the quality score does not count. An unscored run keeps its checks + for diagnosis under ``unscored_checks``, also a dict (see ``_quality_checks``). + """ + if not run.checks: + return _quality_checks(run) + return { + **_quality_checks(run), + **({"unscored_checks": run.checks} if run.harness_error else {}), + "check_details": run.check_details, + **({"diagnostics": run.diagnostics} if run.diagnostics else {}), + **({"harness_error": run.harness_error} if run.harness_error else {}), + } + + def evaluate_agentic_metric_skill( host: str, token: str, @@ -571,7 +1061,7 @@ def evaluate_agentic_metric_skill( ) # Before the Langfuse scores, so an ERROR run gets its marker beside its usual scores. - run_labels = label_runs(summary.run_results, passed=lambda r: r.metric_created and r.maql_correct) + run_labels = label_runs(summary.run_results, passed=lambda r: r.passed, judge_error=lambda r: r.harness_error) if langfuse is not None and dataset_item_id: # Pinned on the calling thread: a deferred poll must not widen its query window. @@ -582,22 +1072,23 @@ def _write_scores(ctx: RunTraceContext) -> None: for run_idx, run in enumerate(summary.run_results): pt = ctx.trace(run.conversation_id) + strict = run.checks or {"metric_created": run.metric_created, "maql_correct": run.maql_correct} with ctx.observe( pt, run_idx, conversation_id=run.conversation_id, - output={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, + output=strict, ) as tid: if (run_error := run_labels.error_label(run_idx)) is not None: log_eval_error_score(ctx, tid, run_error) - ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") - ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") + for name, ok in {**strict, **run.diagnostics}.items(): + ctx.score(tid, name=name, value=float(ok), data_type="BOOLEAN") ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, - strict_checks={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, + strict_checks=_quality_checks(run), latency_sec=pt.latency if pt else None, cost_usd=pt.total_cost if pt else None, ) @@ -619,12 +1110,12 @@ def _write_scores(ctx: RunTraceContext) -> None: item_timings = sum_timings([r.timings for r in summary.run_results]) - runs_passed = sum(1 for r in summary.run_results if r.metric_created and r.maql_correct) + runs_passed = sum(1 for r in summary.run_results if r.passed) runs_effective = len(summary.run_results) run_latencies_s = [r.run_latency_s for r in summary.run_results] best = summary.best - expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] + expected_outputs_list: list[dict] = parse_expectation(expected_output).candidates def _run_detail(run: MetricRunResult) -> dict: """The diagnostic fields for ONE run, shared by the best run and every failing one. @@ -633,8 +1124,7 @@ def _run_detail(run: MetricRunResult) -> dict: for this kind that means the MAQL the run actually produced, which is the whole diagnosis. """ return { - "metric_created": run.metric_created, - "maql_correct": run.maql_correct, + **_verdict_fields(run), "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], "actual_maql": run.actual_maql, # Why the loop stopped. metric_created=False alone cannot tell a refusal from a @@ -651,6 +1141,11 @@ def _failed_checks(run: MetricRunResult) -> list[dict]: so ``metric_created`` alone is named. """ checks: list[FailedCheck] = [] + if run.checks: + for name, ok in run.checks.items(): + if not ok and not (name == "maql_correct" and not run.metric_created): + checks.append(FailedCheck(name, expected=True, actual=run.check_details.get(name, False))) + return failed_check_records(checks) if not run.metric_created: checks.append(FailedCheck("metric_created", expected=True, actual=False)) elif not run.maql_correct: @@ -666,8 +1161,7 @@ def _failed_checks(run: MetricRunResult) -> list[dict]: detail = agentic_detail( best.tool_call_events, best.reasoning_step_events, - metric_created=best.metric_created, - maql_correct=best.maql_correct, + **_verdict_fields(best), expected_maql_candidates=[c.get("maql", "") for c in expected_outputs_list], actual_maql=best.actual_maql, # Why the loop stopped. metric_created=False alone cannot tell a refusal from a run @@ -688,12 +1182,14 @@ def _failed_checks(run: MetricRunResult) -> list[dict]: if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): gate_note = failure_message_head(gate, run_labels, runs_passed, failed_runs) candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) + failing = [name for name, ok in best.checks.items() if not ok] + checks_str = f" Failed checks: {', '.join(failing)}; details: {best.check_details}." if failing else "" raise_agentic_failure( MetricSkillAssertionError, f"Metric skill assertion failed. {gate_note} " f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. " f"Expected MAQL (candidates): {candidates_str}. " - f"Actual MAQL: {best.actual_maql}.", + f"Actual MAQL: {best.actual_maql}.{checks_str}", harness_error=run_labels.harness_error_record, shadow=run_labels.shadow(gate), reasoning_steps=best.reasoning_steps, diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill_options.py b/packages/gooddata-eval/tests/test_agentic_metric_skill_options.py new file mode 100644 index 000000000..dcc0c8b9a --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill_options.py @@ -0,0 +1,1442 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +"""The options shape of a metric-skill expectation: parsing, matchers, and the checks a run publishes.""" + +import json +import sys +import types +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_api_client.exceptions import ApiException, NotFoundException +from gooddata_eval.core.agentic import _metric_workspace as mw +from gooddata_eval.core.agentic import metric_skill as metric_skill_mod +from gooddata_eval.core.agentic._metric_expectation import ( + FieldSpec, + UpdateSpec, + field_mismatches, + first_proposed_maql, + format_matches_kind, + missing_mentions, + parse_expectation, + substitute_placeholders, + update_mismatches, +) +from gooddata_eval.core.agentic._metric_workspace import WorkspaceReadError, last_suggestions +from gooddata_eval.core.agentic.metric_skill import ( + METRIC_CONDITIONAL_SCORES, + METRIC_DIAGNOSTIC_SCORES, + AgenticMetricSummary, + MetricSkillAssertionError, + _match_expected_metrics, + evaluate_agentic_metric_skill, + run_agentic_metric_skill, +) +from gooddata_eval.core.chat.sse_client import ChatError +from gooddata_eval.core.evaluators._llm_judge import JudgeVerdict +from gooddata_eval.core.models import ChatResult, LoopExit, ToolCallEvent + +# Shapes trimmed from the 2026-11-19 probe on an ecommerce_demo copy. +_NET_SALES = { + "title": "Net Sales", + "description": "The sum of all transacted orders adjusted by discounts and returns", + "tags": [], + "content": {"format": "$#,##0.00", "maql": "SELECT {metric/total_sales} - {metric/total_discounts}"}, +} +_AVG_COST = { + "title": "Average Product Cost", + "description": "", + "tags": None, + "content": {"format": "$#,##0.00", "maql": "SELECT AVG({fact/order_unit_cost})"}, +} + + +# --- parsing ---------------------------------------------------------------------------- + + +def test_a_candidate_list_is_the_original_shape() -> None: + spec = parse_expectation([{"maql": "SELECT 1"}, {"maql": "SELECT 2"}]) + assert spec.legacy and not spec.needs_workspace + assert [c["maql"] for c in spec.candidates] == ["SELECT 1", "SELECT 2"] + + +def test_a_candidate_dict_with_title_and_format_stays_the_original_shape() -> None: + # metric_skill items carry title/format beside the MAQL; they must not turn into field checks. + spec = parse_expectation({"maql": "SELECT 1", "title": "X", "format": "#,##0", "tags": ["a"]}) + assert spec.legacy and spec.expected_fields is None + + +def test_options_accept_plain_maql_strings_as_candidates() -> None: + spec = parse_expectation({"candidates": ["SELECT 1"], "expect_executes": True}) + assert not spec.legacy and spec.candidates == [{"maql": "SELECT 1"}] + + +@pytest.mark.parametrize( + ("raw", "message"), + [ + ({"candidates": ["SELECT 1"], "expect_no_creat": True}, "unknown key"), + ({"candidates": ["SELECT 1"], "confirmation_turn": 1}, "at least 2"), + ({"candidates": ["SELECT 1"], "confirmation_turn": True}, "at least 2"), + ({"candidates": ["SELECT 1"], "confirmation_turn": 3, "expect_confirmation_stop": True}, "not both"), + ({"candidates": ["SELECT 1"], "expected_fields": {"format_kind": "money"}}, "format_kind"), + ({"candidates": ["SELECT 1"], "expected_fields": {"colour": "red"}}, "unknown key"), + ({"candidates": ["SELECT 1"], "expect_no_create": True}, "cannot be combined"), + ({"expect_no_create": True, "expected_mentions": []}, "non-empty"), + ({"candidates": ["SELECT 1"], "expected_mentions": ["ARPU"]}, "needs expect_no_create"), + ({"candidates": ["SELECT 1"], "run_all_replies": True}, "needs user_replies"), + ({"expected_metrics": [["SELECT 1"], ["SELECT 2"]]}, "needs user_replies"), + ({"expected_metrics": [["SELECT {metric/@1}"], ["SELECT 2"]], "user_replies": ["yes"]}, "not an earlier entry"), + ({"expected_update": {"title": "X"}}, "needs a metric_id"), + ({"expected_update": {"metric_id": "m"}}, "at least one field"), + ({"expected_update": {"metric_id": "m", "colour": "red"}}, "unknown key"), + ({"user_replies": ["yes"]}, "checks nothing"), + ({"candidates": [{"title": "no maql"}]}, "non-empty MAQL"), + ({"candidates": ["SELECT 1"], "expect_executes": "yes"}, "true or false"), + ], +) +def test_a_malformed_options_shape_fails_loudly(raw: Any, message: str) -> None: + with pytest.raises(ValueError, match=message): + parse_expectation(raw) + + +def test_expect_confirmation_stop_means_no_write_in_turn_one() -> None: + assert parse_expectation({"candidates": ["SELECT 1"], "expect_confirmation_stop": True}).confirmation_turn == 2 + + +def test_a_malformed_expectation_raises_before_any_request() -> None: + with patch("gooddata_eval.core.agentic.metric_skill.ChatClient") as chat_cls, pytest.raises(ValueError): + run_agentic_metric_skill("http://h", "t", "ws", "q", {"candidates": ["SELECT 1"], "typo": 1}) + chat_cls.assert_not_called() + + +# --- matchers --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("fmt", "kind", "expected"), + [ + ("0.0%", "percent", True), + ("#,##0", "percent", False), + ("$#,##0", "currency", True), + ("€#,##0", "currency", True), + ("#,##0", "currency", False), + ("#,##0", "integer", True), + ("$#,##0", "integer", True), + ("#,##0.00", "integer", False), + ("#,##0.00", "decimal", True), + ("0.00%", "decimal", True), + ], +) +def test_format_kinds(fmt: str, kind: str, expected: bool) -> None: + assert format_matches_kind(fmt, kind) is expected + + +def test_field_matching_ignores_case_and_spacing_of_the_title_and_treats_tags_as_a_set() -> None: + stored = {"title": "Median Line cost", "tags": ["kpi", "finance"], "content": {"format": "#,##0.00"}} + spec = FieldSpec(title="Median Line Cost", tags=frozenset({"Finance", "KPI"}), format_kind="decimal") + assert field_mismatches(spec, "median_line_cost", stored) == [] + + +def test_field_matching_names_every_wrong_field() -> None: + stored = {"title": "A", "description": "x", "tags": ["sales"], "content": {"format": "#,##0"}} + spec = FieldSpec( + title="B", metric_id="b", format="$#,##0.00", description_contains=("refund",), tags_include=frozenset({"q4"}) + ) + found = field_mismatches(spec, "a", stored) + assert [m.split(":")[0] for m in found] == ["metric_id", "title", "format", "description", "tags"] + + +def test_field_matching_reports_a_missing_metric() -> None: + assert field_mismatches(FieldSpec(title="X"), "gone", None) == ["metric 'gone' is not stored in the workspace"] + + +def test_an_update_that_resets_the_format_is_caught() -> None: + # GDAI-2838: a rename sends no format, and the tool writes #,##0 over $#,##0.00. + after = {**_AVG_COST, "title": "Avg Unit Cost", "content": {**_AVG_COST["content"], "format": "#,##0"}} + spec = UpdateSpec(metric_id="average_product_cost", fields=FieldSpec(title="Avg Unit Cost")) + assert update_mismatches(spec, _AVG_COST, after) == [ + "format changed although the request did not ask for it: '$#,##0.00' -> '#,##0'" + ] + + +def test_an_update_that_changes_only_what_it_was_asked_passes() -> None: + after = {**_AVG_COST, "content": {**_AVG_COST["content"], "format": "$#,##0"}} + spec = UpdateSpec(metric_id="average_product_cost", fields=FieldSpec(format="$#,##0")) + assert update_mismatches(spec, _AVG_COST, after) == [] + + +def test_an_update_can_require_new_maql() -> None: + after = {**_AVG_COST, "content": {**_AVG_COST["content"], "maql": "SELECT AVG({fact/order_unit_cost}) WITHOUT PF"}} + spec = UpdateSpec( + metric_id="average_product_cost", fields=FieldSpec(), maql=("SELECT AVG({fact/order_unit_cost}) WITHOUT PF",) + ) + assert update_mismatches(spec, _AVG_COST, after) == [] + + +def test_placeholders_take_the_ids_matched_for_earlier_entries() -> None: + assert substitute_placeholders("SELECT {metric/@0} - {metric/@1}", ["a", "b"]) == "SELECT {metric/a} - {metric/b}" + assert substitute_placeholders("SELECT {metric/@1}", ["a"]) == "SELECT {metric/@1}" + + +def test_several_metrics_match_in_order_and_a_dependent_one_uses_their_ids() -> None: + writes = [ + {"metric_id": "total_return_paid", "maql": "SELECT SUM({fact/return_unit_paid_amount})"}, + {"metric_id": "total_return_cost", "maql": "SELECT SUM({fact/return_unit_cost})"}, + {"metric_id": "return_margin", "maql": "SELECT {metric/total_return_paid} - {metric/total_return_cost}"}, + ] + expected = ( + ("SELECT SUM({fact/return_unit_paid_amount})",), + ("SELECT SUM({fact/return_unit_cost})",), + ( + "SELECT {metric/@0} - {metric/@1}", + "SELECT SUM({fact/return_unit_paid_amount}) - SUM({fact/return_unit_cost})", + ), + ) + assert _match_expected_metrics(expected, writes) == ["total_return_paid", "total_return_cost", "return_margin"] + assert _match_expected_metrics(expected, writes[:2]) == ["total_return_paid", "total_return_cost", None] + + +def test_the_first_proposal_is_the_first_formula_in_a_code_span() -> None: + texts = ["I need one detail first.", "**Return Units**\n- Formula: `SELECT SUM({fact/return_unit_quantity})`\n"] + assert first_proposed_maql(texts) == "SELECT SUM({fact/return_unit_quantity})" + assert first_proposed_maql(["no formula here"]) is None + + +def test_mentions_are_found_in_any_reply_whatever_the_case() -> None: + assert missing_mentions(("ARPU", "gross_profit_margin"), ["Use {metric/arpu}.", "nothing"]) == [ + "gross_profit_margin" + ] + + +def test_last_suggestions_reads_the_latest_item_that_carries_them() -> None: + items = [ + {"content": {"type": "multipart", "suggestions": {"actions": [{"query": "old"}]}}}, + {"content": {"type": "multipart"}}, + {"content": {"type": "multipart", "suggestions": {"actions": [{"query": "new"}]}}}, + {"content": "text"}, + ] + assert last_suggestions(items) == {"actions": [{"query": "new"}]} + assert last_suggestions([]) is None + + +# --- runs ------------------------------------------------------------------------------- + + +def _turn(text: str, *writes: dict, parts: list | None = None) -> ChatResult: + return ChatResult.model_validate( + { + "textResponse": text, + "toolCallEvents": [ + {"functionName": "create_metric", "functionArguments": "{}", "result": json.dumps({"data": w})} + for w in writes + ], + "unhandledParts": parts or [], + } + ) + + +class _FakeWorkspace: + """Stands in for MetricWorkspace: an in-memory metric store the test edits as the agent would.""" + + def __init__(self, metrics: dict[str, dict]) -> None: + self.metrics = {k: json.loads(json.dumps(v)) for k, v in metrics.items()} + self.restored: list[tuple[str, bool]] = [] + self.items: list[dict] = [] + self.execute_error: str | None = None + self.read_error: str | None = None + self.snapshot_error: str | None = None + self.restore_fails = False + self.unrestored: list[str] = [] + + def list_metrics(self) -> dict[str, dict]: + if self.snapshot_error: + raise WorkspaceReadError(self.snapshot_error) + return dict(self.metrics) + + def get_metric(self, metric_id: str) -> dict | None: + if self.read_error: + raise WorkspaceReadError(self.read_error) + return self.metrics.get(metric_id) + + def restore_metric(self, metric_id: str, attributes: dict, *, exists: bool) -> bool: + self.restored.append((metric_id, exists)) + if self.restore_fails: + return False + self.metrics[metric_id] = attributes + return True + + def execute_metric(self, metric_id: str) -> str | None: + return self.execute_error + + def conversation_items(self, conversation_id: str) -> list[dict]: + if self.read_error: + raise WorkspaceReadError(self.read_error) + return self.items + + def close(self) -> None: + pass + + +@contextmanager +def _run_env( + turns: Sequence[ChatResult | Exception], workspace: _FakeWorkspace, *, on_send: Callable[[str], None] | None = None +) -> Iterator[tuple[MagicMock, MagicMock]]: + """Patch the chat client to answer ``turns`` in order (an exception is raised), and the other collaborators.""" + client = MagicMock() + client.create_conversation.return_value = "conv-1" + answers = iter(turns) + + def _send(_conversation_id: str, message: str) -> ChatResult: + if on_send: + on_send(message) + answer = next(answers) + if isinstance(answer, Exception): + raise answer + return answer + + client.send_message.side_effect = _send + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=client), + patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk"), + patch("gooddata_eval.core.agentic.metric_skill.MetricWorkspace", return_value=workspace), + patch("gooddata_eval.core.agentic.metric_skill.generate_simulated_response") as sim, + ): + yield client, sim + + +def _run(expected: dict, max_iterations: int = 7) -> AgenticMetricSummary: + return run_agentic_metric_skill("http://h", "t", "ws", "q", expected, k=1, max_iterations=max_iterations) + + +def test_the_original_shape_reads_no_workspace() -> None: + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient") as chat_cls, + patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk"), + patch("gooddata_eval.core.agentic.metric_skill.MetricWorkspace") as ws_cls, + ): + chat_cls.return_value.send_message.return_value = _turn("ok", {"maql": "SELECT 1", "metric_id": "m"}) + summary = run_agentic_metric_skill("http://h", "t", "ws", "q", [{"maql": "SELECT 1"}], k=1, max_iterations=1) + ws_cls.assert_not_called() + assert summary.best.checks == {} and summary.pass_at_k + + +def test_scripted_replies_go_out_in_order_and_the_simulated_user_stays_silent() -> None: + sent: list[str] = [] + ws = _FakeWorkspace({}) + turns = [_turn("Proposal. Should I create it?"), _turn("Understood, I won't create it.")] + with _run_env(turns, ws, on_send=sent.append) as (_, sim): + summary = _run({"expect_no_create": True, "user_replies": ["No, don't create it."]}) + assert sent == ["q", "No, don't create it."] + sim.assert_not_called() + assert summary.best.checks == {"metric_not_created": True} + assert summary.pass_at_k + + +def test_a_write_fails_the_no_create_check_and_a_missing_mention_fails_reuse() -> None: + ws = _FakeWorkspace({}) + turns = [_turn("Created it.", {"metric_id": "rev_per_customer", "maql": "SELECT 1", "created_new": True})] + with _run_env(turns, ws): + summary = _run({"expect_no_create": True, "expected_mentions": ["arpu"]}) + assert summary.best.checks == {"metric_not_created": False, "metric_reuse_named": False} + assert summary.best.check_details["metric_reuse_named"] == {"missing": ["arpu"]} + assert not summary.pass_at_k + + +def test_a_single_turn_item_without_replies_stops_after_the_first_answer() -> None: + ws = _FakeWorkspace({}) + with _run_env([_turn("A matching metric already exists: {metric/arpu}.")], ws) as (client, sim): + summary = _run({"expect_no_create": True, "expected_mentions": ["arpu"]}) + assert client.send_message.call_count == 1 + sim.assert_not_called() + assert summary.pass_at_k + + +def test_a_write_before_the_confirmation_turn_fails() -> None: + # D12: the reply changes the request; the agent must propose again and write only after "yes" (turn 3). + ws = _FakeWorkspace({}) + write = {"metric_id": "m", "maql": "SELECT AVG({fact/rating})", "created_new": True} + turns = [_turn("Proposal?"), _turn("Created.", write)] + with _run_env(turns, ws): + summary = _run( + {"candidates": ["SELECT AVG({fact/rating})"], "confirmation_turn": 3, "user_replies": ["Use avg.", "Yes."]} + ) + assert summary.best.checks["metric_confirmed_first"] is False + assert summary.best.checks["maql_correct"] is True + + +def test_an_overwrite_is_restored_from_the_snapshot_and_fails_no_overwrite() -> None: + ws = _FakeWorkspace({"net_sales": _NET_SALES}) + + def _overwrite(message: str) -> None: + if message == "Yes, go ahead.": + ws.metrics["net_sales"] = {**_NET_SALES, "content": {"format": "#,##0", "maql": "SELECT 1"}} + + write = {"metric_id": "net_sales", "maql": "SELECT 1", "created_new": False} + turns = [_turn("Net Sales exists. Replace it?"), _turn("Updated.", write)] + with _run_env(turns, ws, on_send=_overwrite): + summary = _run({"expect_no_overwrite": True, "user_replies": ["Yes, go ahead."]}) + assert summary.best.checks == {"metric_no_overwrite": False} + assert ws.restored == [("net_sales", True)] + assert ws.metrics["net_sales"] == _NET_SALES + + +def test_run_all_replies_keeps_going_after_a_write_and_judges_the_stored_metric() -> None: + ws = _FakeWorkspace({}) + stored = { + "title": "Lowest Return Paid", + "description": "Smallest amount refunded for one unit", + "tags": ["returns"], + "content": {"format": "$#,##0.00", "maql": "SELECT MIN({fact/return_unit_paid_amount})"}, + } + write = { + "metric_id": "lowest_return_paid", + "maql": "SELECT MIN({fact/return_unit_paid_amount})", + "title": "Lowest Return Paid", + } + + def _store(message: str) -> None: + if message == "Yes, go ahead.": + ws.metrics["lowest_return_paid"] = stored + + turns = [ + _turn("Proposal?"), + _turn("Created.", write), + _turn("Change it like this?"), + _turn("Updated.", {**write, "created_new": False}), + ] + with _run_env(turns, ws, on_send=_store) as (client, _): + summary = _run( + { + "candidates": ["SELECT MIN({fact/return_unit_paid_amount})"], + "user_replies": ["Yes, create it.", "Change its description.", "Yes, go ahead."], + "run_all_replies": True, + "expected_fields": { + "description": "Smallest amount refunded for one unit", + "tags": ["returns"], + "format": "$#,##0.00", + }, + } + ) + assert client.send_message.call_count == 4 + assert summary.best.checks == {"metric_created": True, "maql_correct": True, "metric_fields_correct": True} + + +def test_an_update_check_catches_a_reset_format_and_the_metric_is_restored() -> None: + ws = _FakeWorkspace({"average_product_cost": _AVG_COST}) + + def _rename(message: str) -> None: + if message == "Yes, go ahead.": + ws.metrics["average_product_cost"] = { + **_AVG_COST, + "title": "Avg Unit Cost", + "content": {**_AVG_COST["content"], "format": "#,##0"}, + } + + write = {"metric_id": "average_product_cost", "maql": "SELECT AVG({fact/order_unit_cost})", "created_new": False} + turns = [_turn("Rename it?"), _turn("Renamed.", write)] + with _run_env(turns, ws, on_send=_rename): + summary = _run( + { + "expected_update": {"metric_id": "average_product_cost", "title": "Avg Unit Cost"}, + "user_replies": ["Yes, go ahead."], + } + ) + assert summary.best.checks == {"metric_updated": False} + assert "format changed" in summary.best.check_details["metric_updated"][0] + assert ws.metrics["average_product_cost"] == _AVG_COST + + +def test_several_metrics_are_awaited_across_replies() -> None: + ws = _FakeWorkspace({}) + first = {"metric_id": "a", "maql": "SELECT MAX({fact/order_unit_price})"} + second = {"metric_id": "b", "maql": "SELECT MIN({fact/order_unit_cost})"} + turns = [_turn("Both?"), _turn("Created one.", first), _turn("Created the other.", second)] + with _run_env(turns, ws) as (client, _): + summary = _run( + { + "expected_metrics": [["SELECT MAX({fact/order_unit_price})"], ["SELECT MIN({fact/order_unit_cost})"]], + "user_replies": ["Yes, create them all.", "Please create the remaining ones too."], + } + ) + assert client.send_message.call_count == 3 + assert summary.best.checks == {"metric_all_created": True} + + +def test_clarification_rubric_execution_suggestions_and_first_proposal() -> None: + ws = _FakeWorkspace({}) + ws.items = [ + {"content": {"suggestions": {"actions": [{"label": "Chart", "query": "Show Conversion Rate on a chart"}]}}} + ] + write = { + "metric_id": "conversion_rate", + "maql": "SELECT {metric/net_orders} / {metric/of_orders}", + "title": "Conversion Rate", + } + question_part = [ + {"type": "clarifyingQuestions", "questions": [{"question": "Which conversion?", "options": ["a", "b"]}]} + ] + turns = [ + _turn("Conversion could mean two things.", parts=question_part), + _turn("**Conversion Rate**\n- Formula: `SELECT {metric/net_orders} / {metric/of_orders}`\nCreate it?"), + _turn("Created.", write), + ] + with ( + _run_env(turns, ws), + patch("gooddata_eval.core.agentic.metric_skill.LLMJudge"), + patch( + "gooddata_eval.core.agentic.metric_skill._judge_rubric", return_value=JudgeVerdict(True, "fine") + ) as judge, + ): + summary = _run( + { + "candidates": ["SELECT {metric/net_orders} / {metric/of_orders}"], + "user_replies": ["Processed orders over all orders.", "Yes, create it."], + "expect_clarification": True, + "rubric": "Asks what a conversion is before proposing.", + "expect_executes": True, + "expect_suggestions": True, + "score_first_proposal": True, + } + ) + judge.assert_called_once() + assert summary.best.checks == { + "metric_created": True, + "maql_correct": True, + "metric_clarified": True, + "metric_rubric": True, + "metric_executes": True, + "metric_suggestions": True, + } + assert summary.best.diagnostics == {"metric_first_proposal_correct": True} + + +def test_a_metric_that_does_not_compute_fails_the_execution_check() -> None: + ws = _FakeWorkspace({}) + ws.execute_error = "execute: HTTP 400 invalid MAQL" + with _run_env([_turn("Created.", {"metric_id": "m", "maql": "SELECT 1"})], ws): + summary = _run({"candidates": ["SELECT 1"], "expect_executes": True}) + assert summary.best.checks["metric_executes"] is False + assert summary.best.check_details["metric_executes"] == "execute: HTTP 400 invalid MAQL" + + +def test_a_removed_metric_fails_no_deletion_and_is_recreated() -> None: + ws = _FakeWorkspace({"net_sales": _NET_SALES}) + + def _delete(_message: str) -> None: + ws.metrics.pop("net_sales", None) + + with _run_env([_turn("I can't delete metrics.")], ws, on_send=_delete): + summary = _run({"expect_no_create": True, "expect_no_deletion": ["net_sales"]}) + assert summary.best.checks == {"metric_not_created": True, "metric_none_deleted": False} + assert ws.restored == [("net_sales", False)] + + +def test_every_published_check_name_is_listed_for_the_report() -> None: + names = { + "metric_not_created", + "metric_reuse_named", + "metric_no_overwrite", + "metric_clarified", + "metric_confirmed_first", + "metric_rubric", + "metric_fields_correct", + "metric_updated", + "metric_all_created", + "metric_none_deleted", + "metric_executes", + "metric_suggestions", + "metric_scored", + } + assert set(METRIC_CONDITIONAL_SCORES) == names + assert METRIC_DIAGNOSTIC_SCORES == ("metric_first_proposal_correct",) + + +def _evaluate( + expected: dict, turns: Sequence[ChatResult | Exception], ws: _FakeWorkspace +) -> tuple[Any, dict[str, Any]]: + """Run evaluate_agentic_metric_skill with Langfuse captured; returns (outcome or exception, score dict).""" + result, ctx = _evaluate_ctx(expected, turns, ws) + return result, {c.kwargs["name"]: c.kwargs["value"] for c in ctx.score.call_args_list} + + +def _evaluate_ctx(expected: dict, turns: Sequence[ChatResult | Exception], ws: _FakeWorkspace) -> tuple[Any, MagicMock]: + """Like ``_evaluate``, but returns the Langfuse context the scores were written to.""" + captured: dict[str, Any] = {} + + def _capture(_submit: Any, _identity: Any, **kwargs: Any) -> None: + captured["write_scores"] = kwargs["write_scores"] + + with ( + _run_env(turns, ws), + patch("gooddata_eval.core.agentic.metric_skill.submit_trace_scoring", _capture), + ): + try: + result = evaluate_agentic_metric_skill( + "http://h", "t", "ws", "q", expected, k=1, langfuse=MagicMock(), dataset_item_id="item-1" + ) + except MetricSkillAssertionError as exc: + result = exc + ctx = MagicMock() + captured["write_scores"](ctx) + return result, ctx + + +def test_an_options_item_publishes_only_the_checks_it_opts_into() -> None: + ws = _FakeWorkspace({}) + _, scores = _evaluate({"expect_no_create": True, "expected_mentions": ["arpu"]}, [_turn("Use {metric/arpu}.")], ws) + assert scores["metric_not_created"] == 1.0 + assert scores["metric_reuse_named"] == 1.0 + assert "metric_created" not in scores and "maql_correct" not in scores + + +def test_a_diagnostic_is_published_but_does_not_gate() -> None: + ws = _FakeWorkspace({}) + turns = [_turn("Formula: `SELECT 2`. Create it?"), _turn("Created.", {"metric_id": "m", "maql": "SELECT 1"})] + result, scores = _evaluate( + {"candidates": ["SELECT 1"], "user_replies": ["Make it SELECT 1."], "score_first_proposal": True}, turns, ws + ) + assert scores["metric_first_proposal_correct"] == 0.0 + assert result.runs_passed == 1 + + +def test_a_failing_options_item_names_its_failed_checks() -> None: + ws = _FakeWorkspace({}) + turns = [_turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "created_new": True})] + result, scores = _evaluate({"expect_no_create": True}, turns, ws) + assert scores["metric_not_created"] == 0.0 + assert "Failed checks: metric_not_created" in str(result) + assert result.failed_runs[0]["failed_checks"][0]["name"] == "metric_not_created" + + +# --- review fixes ----------------------------------------------------------------------- + + +def _fake_sdk(pages: dict[int, list[dict]] | None = None, get: Callable[..., Any] | None = None) -> MagicMock: + """An SDK whose entity API serves metric pages (as plain dicts) and single metrics.""" + sdk = MagicMock() + requested: list[int] = [] + + def _list(_ws: str, *, page: int, size: int, _check_return_type: bool) -> types.SimpleNamespace: + requested.append(page) + return types.SimpleNamespace(data=(pages or {}).get(page, []), included=[]) + + sdk._client.entities_api.get_all_entities_metrics.side_effect = _list + if get is not None: + sdk._client.entities_api.get_entity_metrics.side_effect = get + sdk.requested = requested + return sdk + + +def test_metrics_are_listed_through_the_sdk_pager_which_stops_at_a_short_page() -> None: + full = [{"id": f"m{i}", "attributes": {"title": str(i), "content": {"maql": "SELECT 1"}}} for i in range(500)] + last = {"title": "L", "content": {"maql": "SELECT 2"}} + sdk = _fake_sdk({0: full, 1: [{"id": "last", "attributes": last}]}) + metrics = mw.MetricWorkspace("http://h", "t", "ws", sdk).list_metrics() + assert sdk.requested == [0, 1] + assert len(metrics) == 501 and metrics["last"] == last + + +def test_a_missing_metric_reads_as_none_and_another_failure_as_a_read_error() -> None: + def _get(_ws: str, metric_id: str, _check_return_type: bool) -> types.SimpleNamespace: + if metric_id == "gone": + raise NotFoundException(status=404) + if metric_id == "broken": + raise ApiException(status=500) + # Like the real client: a document model whose data is a plain dict. + return types.SimpleNamespace(data={"id": metric_id, "attributes": {"title": "T"}}) + + ws = mw.MetricWorkspace("http://h", "t", "ws", _fake_sdk(get=_get)) + assert ws.get_metric("ok") == {"title": "T"} + assert ws.get_metric("gone") is None + with pytest.raises(WorkspaceReadError): + ws.get_metric("broken") + + +def test_a_restore_writes_every_writable_attribute_back() -> None: + sent: list[dict] = [] + + def _handler(request: mw.httpx.Request) -> mw.httpx.Response: + sent.append({"method": request.method, "body": json.loads(request.content)}) + return mw.httpx.Response(200, json={}) + + ws = mw.MetricWorkspace("http://h", "t", "ws", _fake_sdk()) + ws._client = mw.httpx.Client(transport=mw.httpx.MockTransport(_handler)) + attributes = { + **_NET_SALES, + "isHidden": True, + "isHiddenFromKda": True, + "conditionalFormatting": {"rules": []}, + "createdAt": "2026-01-01", + "areRelationsValid": True, + } + assert ws.restore_metric("net_sales", attributes, exists=True) + written = sent[0]["body"]["data"]["attributes"] + assert sent[0]["method"] == "PUT" + assert written == { + "title": _NET_SALES["title"], + "description": _NET_SALES["description"], + "tags": [], + "content": _NET_SALES["content"], + "isHidden": True, + "isHiddenFromKda": True, + "conditionalFormatting": {"rules": []}, + } + + +def test_an_unreadable_conversation_is_a_harness_error_not_an_agent_failure() -> None: + ws = _FakeWorkspace({}) + ws.read_error = "HTTP 500" + with _run_env([_turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "title": "M"})], ws): + summary = _run({"candidates": ["SELECT 1"], "expect_suggestions": True}) + assert summary.best.checks["metric_scored"] is False + assert summary.best.harness_error == "metric_scored: HTTP 500" + + +def test_a_harness_error_labels_the_run_error() -> None: + ws = _FakeWorkspace({}) + ws.read_error = "HTTP 502" + result, scores = _evaluate( + {"candidates": ["SELECT 1"], "expect_executes": True, "expected_fields": {"title": "M"}}, + [_turn("Created.", {"metric_id": "m", "maql": "SELECT 1"})], + ws, + ) + assert scores["eval_error"] == 1 + assert "Harness error" in str(result) + + +def test_a_failed_snapshot_skips_the_run_instead_of_driving_the_agent() -> None: + ws = _FakeWorkspace({}) + ws.snapshot_error = "could not list metrics: 503" + with _run_env([_turn("never sent")], ws) as (client, _): + summary = _run({"candidates": ["SELECT 1"], "expect_no_overwrite": True}) + client.send_message.assert_not_called() + assert summary.best.exit_reason == LoopExit.NOT_RUN + assert summary.best.harness_error == "snapshot: could not list metrics: 503" + + +def test_editing_a_metric_the_run_created_is_not_an_overwrite() -> None: + ws = _FakeWorkspace({}) + created = {"metric_id": "new_m", "maql": "SELECT 1", "created_new": True} + edited = {**created, "created_new": False} + turns = [_turn("Created.", created), _turn("Change it?"), _turn("Updated.", edited)] + with _run_env(turns, ws): + summary = _run( + { + "candidates": ["SELECT 1"], + "expect_no_overwrite": True, + "user_replies": ["Change its description.", "Yes."], + "run_all_replies": True, + } + ) + assert summary.best.checks["metric_no_overwrite"] is True + assert ws.restored == [] + + +def test_a_confirmation_prompt_is_not_a_clarifying_question() -> None: + ws = _FakeWorkspace({}) + turns = [ + _turn("Here it is: `SELECT 1`. Should I create it?"), + _turn("Created.", {"metric_id": "m", "maql": "SELECT 1"}), + ] + with _run_env(turns, ws): + summary = _run({"candidates": ["SELECT 1"], "expect_clarification": True, "user_replies": ["Yes."]}) + assert summary.best.checks["metric_clarified"] is False + + +def test_a_missing_judge_fails_before_any_request() -> None: + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient") as chat_cls, + patch("gooddata_eval.core.agentic.metric_skill.LLMJudge", side_effect=OSError("OPENAI_API_KEY is required")), + pytest.raises(OSError), + ): + _run({"expect_no_create": True, "rubric": "Says it cannot delete."}) + chat_cls.assert_not_called() + + +def test_a_judge_without_a_verdict_is_a_harness_error() -> None: + ws = _FakeWorkspace({}) + with ( + _run_env([_turn("I can't delete metrics.")], ws), + patch("gooddata_eval.core.agentic.metric_skill.LLMJudge"), + patch( + "gooddata_eval.core.agentic.metric_skill._judge_rubric", + return_value=JudgeVerdict(False, "", error="unreadable"), + ), + ): + summary = _run({"expect_no_create": True, "rubric": "Says it cannot delete."}) + assert summary.best.checks["metric_rubric"] is False + assert summary.best.harness_error == "metric_rubric: judge gave no verdict: unreadable" + + +def test_the_judge_reads_user_and_assistant_turns_in_order_simulated_replies_included() -> None: + ws = _FakeWorkspace({}) + turns = [_turn("Which date?"), _turn("Created.", {"metric_id": "m", "maql": "SELECT 1"})] + with ( + _run_env(turns, ws) as (_, sim), + patch("gooddata_eval.core.agentic.metric_skill.LLMJudge"), + patch("gooddata_eval.core.agentic.metric_skill._judge_rubric", return_value=JudgeVerdict(True, "")) as judge, + ): + sim.return_value = "Use the order date." + _run({"candidates": ["SELECT 1"], "rubric": "Asks which date before proposing."}) + transcript = judge.call_args.args[2] + assert transcript == [ + ("User", "q"), + ("Assistant", "Which date?"), + ("User", "Use the order date."), + ("Assistant", "Created."), + ] + + +def test_a_spent_script_stops_early_rather_than_succeeding() -> None: + ws = _FakeWorkspace({}) + with _run_env([_turn("Which one?"), _turn("Still need to know which one?")], ws): + summary = _run({"expected_fields": {"title": "X"}, "user_replies": ["Yes."]}) + assert summary.best.exit_reason == LoopExit.STOPPED_EARLY + assert summary.best.checks == {"metric_fields_correct": False} + + +def test_a_write_in_a_broken_turn_counts_for_the_confirmation_check() -> None: + ws = _FakeWorkspace({}) + partial = _turn("", {"metric_id": "m", "maql": "SELECT 1", "created_new": True}) + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = ChatError("stream broke", partial_result=partial) + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=client), + patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk"), + patch("gooddata_eval.core.agentic.metric_skill.MetricWorkspace", return_value=ws), + ): + summary = _run({"candidates": ["SELECT 1"], "expect_confirmation_stop": True}) + assert summary.best.checks["metric_confirmed_first"] is False + + +def test_an_options_item_puts_its_checks_at_the_top_of_the_detail() -> None: + ws = _FakeWorkspace({}) + result, _ = _evaluate({"expect_no_create": True}, [_turn("No metric needed.")], ws) + assert result.detail["metric_not_created"] is True + assert "metric_created" not in result.detail and "maql_correct" not in result.detail + + +@pytest.mark.parametrize( + ("raw", "message"), + [ + ({"expect_no_create": True, "expect_executes": True}, "cannot be combined"), + ({"expect_no_create": True, "expect_suggestions": True}, "cannot be combined"), + ({"expected_update": {"metric_id": "m", "title": "T"}, "expect_no_overwrite": True}, "forbids"), + ({"expect_no_deletion": True}, "non-empty string"), + ({"rubric": "x", "expect_executes": True}, "need candidates or expected_fields"), + ], +) +def test_contradicting_options_are_rejected(raw: dict, message: str) -> None: + with pytest.raises(ValueError, match=message): + parse_expectation(raw) + + +@pytest.mark.parametrize("raw", [["SELECT 1"], "SELECT 1", [{"maql": "SELECT 1"}, "SELECT 2"]]) +def test_an_original_shape_with_a_plain_string_fails_loudly(raw: Any) -> None: + with pytest.raises(ValueError, match="plain MAQL strings go under 'candidates'"): + parse_expectation(raw) + + +def test_the_first_proposal_is_read_from_a_fenced_block_too() -> None: + text = "Here is the formula:\n```maql\nSELECT SUM({fact/x})\n```\nCreate it?" + assert first_proposed_maql([text]) == "SELECT SUM({fact/x})" + + +# --- CodeRabbit review and coverage --------------------------------------------------------- + + +@pytest.mark.parametrize( + ("raw", "message"), + [ + ({"candidates": ["SELECT 1"], "rubric": None}, "rubric must be"), + ({"candidates": ["SELECT 1"], "rubric": " "}, "rubric must be"), + ({"candidates": ["SELECT 1"], "expected_fields": {"format_kind": None}}, "format_kind"), + ({"candidates": ["SELECT 1"], "expected_fields": {}}, "non-empty object"), + ({"candidates": ["SELECT 1"], "expected_fields": {"title": ""}}, "non-empty string"), + ({"candidates": []}, "non-empty list"), + ({"expected_metrics": [["SELECT 1"]], "user_replies": ["yes"]}, "at least two"), + ({"candidates": ["SELECT 1"], "expected_update": {"metric_id": "m", "title": "T"}}, "drop candidates"), + ({"expect_no_create": True, "score_first_proposal": True}, "needs candidates"), + ({"confirmation_turn": 2, "user_replies": ["y"], "rubric": ""}, "rubric"), + ], +) +def test_more_malformed_options_fail_loudly(raw: dict, message: str) -> None: + with pytest.raises(ValueError, match=message): + parse_expectation(raw) + + +def test_a_single_candidate_string_and_a_lone_confirmation_turn_are_valid() -> None: + assert parse_expectation({"candidates": "SELECT 1", "expect_executes": True}).candidates == [{"maql": "SELECT 1"}] + spec = parse_expectation({"confirmation_turn": 2, "user_replies": ["Yes"]}) + assert spec.confirmation_turn == 2 and not spec.candidates + with pytest.raises(ValueError, match="checks nothing"): + parse_expectation({"expect_confirmation_stop": False, "user_replies": ["Yes"]}) + + +@pytest.mark.parametrize("kind", ["integer", "decimal", "percent", "currency"]) +def test_a_missing_format_is_no_kind(kind: str) -> None: + assert format_matches_kind("", kind) is False + assert format_matches_kind(" ", kind) is False + + +def test_field_matching_reports_kind_description_and_tags() -> None: + stored = {"title": "A", "description": "x", "tags": ["a"], "content": {"format": ""}} + spec = FieldSpec(format_kind="integer", description="y", tags=frozenset({"b"})) + assert [m.split(":")[0] for m in field_mismatches(spec, "a", stored)] == ["format", "description", "tags"] + + +def test_an_update_needs_a_metric_that_existed_and_still_exists() -> None: + spec = UpdateSpec(metric_id="m", fields=FieldSpec(title="T")) + assert update_mismatches(spec, None, _AVG_COST) == ["metric 'm' did not exist before the run"] + assert update_mismatches(spec, _AVG_COST, None) == ["metric 'm' is not stored in the workspace"] + + +def test_an_update_with_the_wrong_maql_is_reported() -> None: + spec = UpdateSpec(metric_id="m", fields=FieldSpec(), maql=("SELECT 2",)) + assert update_mismatches(spec, _AVG_COST, _AVG_COST)[0].startswith("maql: expected one of ['SELECT 2']") + + +def test_an_update_whose_requested_field_was_already_right_does_not_pass() -> None: + # The agent only replied; the title was already the requested one. + spec = UpdateSpec(metric_id="m", fields=FieldSpec(title="Average Product Cost")) + assert update_mismatches(spec, _AVG_COST, _AVG_COST) == [ + "title did not change although the request asked for it: still 'Average Product Cost'" + ] + + +def test_an_update_that_unhides_a_hidden_metric_is_caught() -> None: + before = {**_AVG_COST, "isHidden": True, "conditionalFormatting": {"rules": [1]}} + after = {**_AVG_COST, "title": "Avg Unit Cost"} + spec = UpdateSpec(metric_id="m", fields=FieldSpec(title="Avg Unit Cost")) + changed = [m.split(" ")[0] for m in update_mismatches(spec, before, after)] + assert changed == ["isHidden", "conditionalFormatting"] + + +def test_several_metrics_are_judged_by_the_latest_write_of_each_id() -> None: + writes = [ + {"metric_id": "a", "maql": "SELECT 1"}, + {"metric_id": "a", "maql": "SELECT 9"}, + {"metric_id": "b", "maql": "SELECT 2"}, + ] + assert _match_expected_metrics((("SELECT 1",), ("SELECT 2",)), writes) == [None, "b"] + + +def test_several_metrics_try_every_assignment() -> None: + writes = [{"metric_id": "a", "maql": "SELECT 1"}, {"metric_id": "b", "maql": "SELECT 2"}] + assert _match_expected_metrics((("SELECT 1", "SELECT 2"), ("SELECT 1",)), writes) == ["b", "a"] + + +def test_run_all_replies_sends_every_reply_although_every_metric_is_there() -> None: + ws = _FakeWorkspace({}) + both = [{"metric_id": "a", "maql": "SELECT 1"}, {"metric_id": "b", "maql": "SELECT 2"}] + turns = [_turn("Both?"), _turn("Created both.", *both), _turn("Anything else?")] + with _run_env(turns, ws) as (client, _): + summary = _run( + { + "expected_metrics": [["SELECT 1"], ["SELECT 2"]], + "user_replies": ["Yes.", "Thanks, that's all."], + "run_all_replies": True, + } + ) + assert client.send_message.call_count == 3 + assert summary.best.checks == {"metric_all_created": True} + + +def test_a_chat_failure_before_any_answer_is_no_proof_of_refusal() -> None: + ws = _FakeWorkspace({}) + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.side_effect = ChatError("stream broke") + with ( + patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=client), + patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk"), + patch("gooddata_eval.core.agentic.metric_skill.MetricWorkspace", return_value=ws), + ): + summary = _run({"expect_no_create": True}) + assert summary.best.checks == {"metric_not_created": False} + assert summary.best.check_details["metric_not_created"] == "the agent never completed an answer" + + +def test_a_judge_provider_failure_is_a_harness_error() -> None: + ws = _FakeWorkspace({}) + with ( + _run_env([_turn("I can't delete metrics.")], ws), + patch("gooddata_eval.core.agentic.metric_skill.LLMJudge"), + patch("gooddata_eval.core.agentic.metric_skill._judge_rubric", side_effect=TimeoutError("judge timed out")), + ): + summary = _run({"expect_no_create": True, "rubric": "Says it cannot delete."}) + assert summary.best.checks["metric_rubric"] is False + assert summary.best.harness_error == "metric_rubric: judge gave no verdict: TimeoutError: judge timed out" + + +def test_the_rubric_judge_gets_the_conversation_in_order() -> None: + judge = MagicMock() + judge.score.return_value = (True, "ok") + verdict = metric_skill_mod._judge_rubric(judge, "criteria", [("User", "q"), ("Assistant", "a")]) + assert verdict.passed + assert judge.score.call_args.kwargs == { + "input": "q", + "expected_output": "criteria", + "actual_output": "User: q\n\nAssistant: a", + } + + +def test_expected_fields_read_the_metric_the_agent_wrote_not_the_named_one() -> None: + ws = _FakeWorkspace({"sales": {"title": "Sales", "content": {"maql": "SELECT 1", "format": "#,##0"}}}) + write = {"metric_id": "wrong_id", "maql": "SELECT 1"} + + def _store(_message: str) -> None: + ws.metrics["wrong_id"] = {"title": "Other", "content": {"maql": "SELECT 1"}} + + with _run_env([_turn("Created.", write)], ws, on_send=_store): + summary = _run({"candidates": ["SELECT 1"], "expected_fields": {"metric_id": "sales", "title": "Sales"}}) + assert summary.best.checks["metric_fields_correct"] is False + assert summary.best.check_details["metric_fields_correct"][0] == "metric_id: expected 'sales', got 'wrong_id'" + + +def test_a_failed_restore_is_a_harness_error_and_stops_later_runs() -> None: + ws = _FakeWorkspace({"net_sales": _NET_SALES}) + ws.restore_fails = True + write = {"metric_id": "net_sales", "maql": "SELECT 1", "created_new": False} + turns = [_turn("Updated.", write), _turn("never sent")] + with _run_env(turns, ws) as (client, _): + summary = run_agentic_metric_skill( + "http://h", "t", "ws", "q", {"expect_no_overwrite": True}, k=2, max_iterations=1 + ) + first, second = summary.run_results + assert first.harness_error == "restore failed for ['net_sales']" + assert second.exit_reason == LoopExit.NOT_RUN and "could not put back" in second.harness_error + assert client.send_message.call_count == 1 + + +def test_a_guarded_metric_that_cannot_be_checked_counts_as_not_restored() -> None: + ws = _FakeWorkspace({"net_sales": _NET_SALES}) + calls = {"n": 0} + + def _get(metric_id: str) -> dict | None: + calls["n"] += 1 + if calls["n"] > 1: + raise WorkspaceReadError("HTTP 502") + return ws.metrics.get(metric_id) + + ws.get_metric = _get + with _run_env([_turn("I can't delete metrics.")], ws): + summary = _run({"expect_no_create": True, "expect_no_deletion": ["net_sales"]}) + assert summary.best.checks["metric_none_deleted"] is True + assert ws.unrestored == ["net_sales"] + + +def test_suggestions_without_a_written_metric_do_not_fail_the_check() -> None: + ws = _FakeWorkspace({}) + ws.items = [{"content": {"suggestions": {"actions": [{"query": "Show something"}]}}}] + with _run_env([_turn("Which one?")], ws): + summary = _run({"candidates": ["SELECT 1"], "expect_suggestions": True}, max_iterations=1) + assert summary.best.checks["metric_suggestions"] is True + + +def test_suggestions_naming_another_metric_fail_the_check() -> None: + ws = _FakeWorkspace({}) + ws.items = [{"content": {"suggestions": {"actions": [{"label": "Chart", "query": "Show Net Sales"}]}}}] + with _run_env([_turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "title": "Margin"})], ws): + summary = _run({"candidates": ["SELECT 1"], "expect_suggestions": True}) + assert summary.best.checks["metric_suggestions"] is False + + +def test_a_silent_agent_ends_the_run() -> None: + ws = _FakeWorkspace({}) + with _run_env([_turn("")], ws): + summary = _run({"candidates": ["SELECT 1"]}) + assert summary.best.exit_reason == LoopExit.AGENT_SILENT + + +def test_a_wrong_maql_on_an_options_item_names_maql_correct() -> None: + ws = _FakeWorkspace({}) + result, _ = _evaluate( + {"candidates": ["SELECT 1"], "expect_executes": True}, + [_turn("Created.", {"metric_id": "m", "maql": "SELECT 2"})], + ws, + ) + assert [c["name"] for c in result.failed_runs[0]["failed_checks"]] == ["maql_correct"] + + +# --- MetricWorkspace against a mocked HTTP transport -------------------------------------------- + + +def _http_workspace( + handler: Callable[[mw.httpx.Request], mw.httpx.Response], sdk: MagicMock | None = None +) -> mw.MetricWorkspace: + ws = mw.MetricWorkspace("http://h", "t", "ws", sdk or _fake_sdk()) + ws._client = mw.httpx.Client(transport=mw.httpx.MockTransport(handler)) + return ws + + +@pytest.fixture +def _no_backoff(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", "0") + monkeypatch.setenv("GOODDATA_EVAL_CHAT_MAX_RETRIES", "2") + + +def test_a_restore_retries_a_transient_status_and_posts_a_removed_metric(_no_backoff: None) -> None: + statuses = iter([503, 201]) + seen: list[str] = [] + + def _handler(request: mw.httpx.Request) -> mw.httpx.Response: + seen.append(request.method) + return mw.httpx.Response(next(statuses), json={}) + + ws = _http_workspace(_handler) + assert ws.restore_metric("net_sales", _NET_SALES, exists=False) + assert seen == ["POST", "POST"] + ws.close() + + +def test_a_restore_that_keeps_failing_returns_false(_no_backoff: None) -> None: + ws = _http_workspace(lambda request: mw.httpx.Response(503, json={})) + assert ws.restore_metric("net_sales", _NET_SALES, exists=True) is False + assert ( + _http_workspace(lambda request: mw.httpx.Response(400, json={})).restore_metric( + "net_sales", _NET_SALES, exists=True + ) + is False + ) + + def _boom(request: mw.httpx.Request) -> mw.httpx.Response: + raise mw.httpx.ReadTimeout("slow", request=request) + + assert _http_workspace(_boom).restore_metric("net_sales", _NET_SALES, exists=True) is False + + +def test_listing_or_reading_metrics_wraps_api_failures() -> None: + sdk = _fake_sdk() + sdk._client.entities_api.get_all_entities_metrics.side_effect = ApiException(status=400) + ws = mw.MetricWorkspace("http://h", "t", "ws", sdk) + with pytest.raises(WorkspaceReadError, match="could not list metrics"): + ws.list_metrics() + + +@pytest.mark.parametrize( + ("row", "message"), + [({"id": "m"}, "unreadable row"), ({"id": "m", "attributes": {"title": "T"}}, "has no MAQL")], +) +def test_an_incomplete_snapshot_row_fails_the_snapshot(row: dict, message: str) -> None: + ws = mw.MetricWorkspace("http://h", "t", "ws", _fake_sdk({0: [row]})) + with pytest.raises(WorkspaceReadError, match=message): + ws.list_metrics() + + +def test_a_metric_lookup_without_attributes_is_a_read_error() -> None: + ws = mw.MetricWorkspace("http://h", "t", "ws", _fake_sdk(get=lambda *a, **k: types.SimpleNamespace(data=None))) + with pytest.raises(WorkspaceReadError, match="no attributes"): + ws.get_metric("m") + + +def _execute_handler( + execute_status: int = 200, execute_body: dict | None = None, result_status: int = 200 +) -> Callable[[mw.httpx.Request], mw.httpx.Response]: + def _handler(request: mw.httpx.Request) -> mw.httpx.Response: + if request.method == "POST": + body = ( + execute_body + if execute_body is not None + else {"executionResponse": {"links": {"executionResult": "r1"}}} + ) + return mw.httpx.Response(execute_status, json=body) + return mw.httpx.Response(result_status, json={"data": [[1]]}) + + return _handler + + +def test_a_metric_that_computes_has_no_rejection() -> None: + assert _http_workspace(_execute_handler()).execute_metric("m") is None + + +@pytest.mark.parametrize(("execute", "result"), [(400, 200), (200, 422)]) +def test_a_platform_rejection_is_the_check_verdict(execute: int, result: int) -> None: + reason = _http_workspace(_execute_handler(execute_status=execute, result_status=result)).execute_metric("m") + assert reason is not None and "HTTP 4" in reason + + +@pytest.mark.parametrize( + "handler", + [ + pytest.param(_execute_handler(execute_status=503), id="execute-503"), + pytest.param(_execute_handler(result_status=502), id="result-502"), + pytest.param(_execute_handler(execute_status=429), id="execute-429"), + pytest.param(_execute_handler(result_status=429), id="result-429"), + pytest.param(_execute_handler(execute_body={"executionResponse": None}), id="malformed"), + ], +) +def test_an_execution_service_fault_is_a_read_error( + handler: Callable[[mw.httpx.Request], mw.httpx.Response], _no_backoff: None +) -> None: + with pytest.raises(WorkspaceReadError, match="could not execute metric"): + _http_workspace(handler).execute_metric("m") + + +def test_conversation_items_accept_items_or_data_and_reject_anything_else() -> None: + items = [{"content": {"suggestions": {"actions": []}}}] + assert _http_workspace(lambda r: mw.httpx.Response(200, json={"items": items})).conversation_items("c") == items + assert _http_workspace(lambda r: mw.httpx.Response(200, json={"data": []})).conversation_items("c") == [] + with pytest.raises(WorkspaceReadError, match="no items list"): + _http_workspace(lambda r: mw.httpx.Response(200, json={"unexpected": []})).conversation_items("c") + with pytest.raises(WorkspaceReadError, match="could not read conversation"): + _http_workspace(lambda r: mw.httpx.Response(500, json={})).conversation_items("c") + + +# --- second review round ---------------------------------------------------------------------- + + +def test_a_failed_restore_never_lets_the_run_pass() -> None: + # Every check passes, the restore fails: the run is unscored, so a single-run item cannot pass its gate. + ws = _FakeWorkspace({"average_product_cost": _AVG_COST}) + ws.restore_fails = True + write = {"metric_id": "average_product_cost", "maql": "SELECT AVG({fact/order_unit_cost})", "created_new": False} + + def _rename(message: str) -> None: + ws.metrics["average_product_cost"] = {**_AVG_COST, "title": "Avg Unit Cost"} + + with _run_env([_turn("Renamed.", write)], ws, on_send=_rename): + summary = _run({"expected_update": {"metric_id": "average_product_cost", "title": "Avg Unit Cost"}}) + run = summary.run_results[0] + assert run.checks == {"metric_updated": True, "metric_scored": False} + assert run.check_details["metric_scored"] == "harness: restore failed for ['average_product_cost']" + assert run.harness_error == "restore failed for ['average_product_cost']" + assert run.passed is False and summary.pass_at_k is False and summary.pass_power_k is False + + +def test_a_failed_restore_labels_the_run_error() -> None: + ws = _FakeWorkspace({"net_sales": _NET_SALES}) + ws.restore_fails = True + write = {"metric_id": "net_sales", "maql": "SELECT 1", "created_new": False} + result, scores = _evaluate({"candidates": ["SELECT 1"]}, [_turn("Updated.", write)], ws) + assert isinstance(result, MetricSkillAssertionError) + assert scores["eval_error"] == 1 + assert "Harness error" in str(result) + + +@pytest.mark.parametrize( + "suggestions", + [{"actions": ["invalid"]}, {"actions": "Show it"}, {"actions": {}}], + ids=["action-not-object", "actions-not-list", "actions-empty-object"], +) +def test_malformed_suggestion_actions_are_a_harness_error(suggestions: dict) -> None: + ws = _FakeWorkspace({}) + ws.items = [{"content": {"suggestions": suggestions}}] + with _run_env([_turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "title": "M"})], ws): + summary = _run({"candidates": ["SELECT 1"], "expect_suggestions": True}) + assert summary.best.checks["metric_scored"] is False + assert "unreadable suggestion actions" in summary.best.harness_error + + +def test_an_execution_response_that_is_not_an_object_is_a_read_error() -> None: + handler = _execute_handler(execute_body={"executionResponse": ["unexpected"]}) + with pytest.raises(WorkspaceReadError, match="no executionResult"): + _http_workspace(handler).execute_metric("m") + + +@pytest.mark.parametrize("failure", ["timeout", "bad-json"]) +def test_an_execution_that_cannot_be_read_is_a_read_error(failure: str) -> None: + def _handler(request: mw.httpx.Request) -> mw.httpx.Response: + if failure == "timeout": + raise mw.httpx.ReadTimeout("slow", request=request) + return mw.httpx.Response(200, content=b"not json") + + with pytest.raises(WorkspaceReadError, match="could not execute metric m"): + _http_workspace(_handler).execute_metric("m") + + +def test_judge_provider_errors_cover_openai_when_it_is_installed_and_transport_always() -> None: + errors = metric_skill_mod._judge_provider_errors() + assert {mw.httpx.HTTPError, TimeoutError, ConnectionError} <= set(errors) + with patch.dict(sys.modules, {"openai": None}): + assert metric_skill_mod._judge_provider_errors() == (mw.httpx.HTTPError, TimeoutError, ConnectionError) + + +def test_a_write_whose_result_is_not_an_object_is_ignored() -> None: + events = [ + ToolCallEvent(function_name="search_objects", function_arguments="{}", result='{"objects": []}'), + ToolCallEvent(function_name="create_metric", function_arguments="{}", result=""), + ToolCallEvent(function_name="create_metric", function_arguments="{}", result='["not", "an", "object"]'), + ToolCallEvent(function_name="create_metric", function_arguments="{}", result='{"data": {"metric_id": "m"}}'), + ] + assert metric_skill_mod._successful_writes(events) == [{"metric_id": "m"}] + + +def test_a_guarded_metric_missing_from_the_snapshot_is_left_alone() -> None: + ws = _FakeWorkspace({}) + assert metric_skill_mod._restore_workspace(ws, {}, [], ("not_there",)) == [] + assert ws.restored == [] + + +def test_a_guarded_metric_that_cannot_be_recreated_is_reported() -> None: + ws = _FakeWorkspace({}) + ws.restore_fails = True + assert metric_skill_mod._restore_workspace(ws, {"net_sales": _NET_SALES}, [], ("net_sales",)) == ["net_sales"] + assert ws.restored == [("net_sales", False)] + + +# --- third review round ----------------------------------------------------------------------- + + +def test_a_rate_limit_is_retried_before_the_metric_is_judged(_no_backoff: None) -> None: + statuses = iter([429, 200]) + + def _handler(request: mw.httpx.Request) -> mw.httpx.Response: + if request.method == "POST": + return mw.httpx.Response(next(statuses), json={"executionResponse": {"links": {"executionResult": "r1"}}}) + return mw.httpx.Response(200, json={"data": [[1]]}) + + assert _http_workspace(_handler).execute_metric("m") is None + + +@pytest.mark.parametrize( + ("result", "computes"), + [ + pytest.param({"json": {"data": []}}, True, id="empty-data"), + pytest.param({"content": b"not json"}, False, id="not-json"), + pytest.param({"json": {"unexpected": 1}}, False, id="no-data-list"), + pytest.param({"json": ["not", "an", "object"]}, False, id="not-an-object"), + ], +) +def test_a_result_must_carry_its_data_list(result: dict, computes: bool) -> None: + def _handler(request: mw.httpx.Request) -> mw.httpx.Response: + if request.method == "POST": + return mw.httpx.Response(200, json={"executionResponse": {"links": {"executionResult": "r1"}}}) + return mw.httpx.Response(200, **result) + + ws = _http_workspace(_handler) + if computes: + assert ws.execute_metric("m") is None + else: + with pytest.raises(WorkspaceReadError, match="could not execute metric m"): + ws.execute_metric("m") + + +def test_the_first_proposal_falls_back_to_the_first_write_not_the_last() -> None: + ws = _FakeWorkspace({}) + turns = [ + _turn("Created.", {"metric_id": "m", "maql": "SELECT 2", "created_new": True}), + _turn("Fixed.", {"metric_id": "m", "maql": "SELECT 1"}), + ] + with _run_env(turns, ws): + summary = _run( + { + "candidates": ["SELECT 1"], + "user_replies": ["Make it SELECT 1."], + "run_all_replies": True, + "score_first_proposal": True, + } + ) + assert summary.best.checks["maql_correct"] is True + assert summary.best.diagnostics == {"metric_first_proposal_correct": False} + assert summary.best.check_details["metric_first_proposal_correct"] == {"proposed": "SELECT 2"} + + +def test_a_guarded_metric_overwritten_and_removed_is_recreated_once_and_not_reported() -> None: + ws = _FakeWorkspace({}) # The run wrote over net_sales, then removed it. + writes = [{"metric_id": "net_sales", "maql": "SELECT 1", "created_new": False}] + assert metric_skill_mod._restore_workspace(ws, {"net_sales": _NET_SALES}, writes, ("net_sales",)) == [] + assert ws.restored == [("net_sales", False)] + assert ws.metrics["net_sales"] == _NET_SALES + + +def test_a_chat_failure_in_a_scripted_turn_fails_the_run_after_a_correct_write() -> None: + ws = _FakeWorkspace({}) + turns: list[ChatResult | Exception] = [ + _turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "created_new": True}), + ChatError("stream broke"), + ] + result, scores = _evaluate( + {"candidates": ["SELECT 1"], "user_replies": ["Now make it hidden."], "run_all_replies": True}, turns, ws + ) + assert isinstance(result, MetricSkillAssertionError) + assert scores["metric_created"] == 1.0 and scores["maql_correct"] == 1.0 + assert "eval_error" not in scores # The agent's chat failed: a FAIL, not a harness ERROR. + assert result.failed_runs[0]["exit_reason"] == LoopExit.CHAT_ERROR.value + + +def test_a_completed_script_without_a_further_write_still_passes() -> None: + ws = _FakeWorkspace({}) + turns = [_turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "created_new": True}), _turn("You're welcome.")] + with _run_env(turns, ws): + summary = _run({"candidates": ["SELECT 1"], "user_replies": ["Thanks."], "run_all_replies": True}) + assert summary.best.exit_reason == LoopExit.STOPPED_EARLY + assert summary.pass_at_k is True + + +def test_an_unscored_run_counts_as_metric_scored_false_alone_for_quality() -> None: + ws = _FakeWorkspace({}) + ws.read_error = "HTTP 502" + result, ctx = _evaluate_ctx( + {"candidates": ["SELECT 1"], "expect_suggestions": True}, + [_turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "title": "M"})], + ws, + ) + assert ctx.quality.call_args.kwargs["strict_checks"] == {"metric_scored": False} + assert [k for k, v in result.detail.items() if isinstance(v, bool)] == ["metric_scored"] + assert result.detail["unscored_checks"] == {"metric_created": True, "maql_correct": True, "metric_scored": False} + + +def test_a_graded_run_is_the_best_run_before_an_unscored_one() -> None: + ws = _FakeWorkspace({}) + sent = {"n": 0} + + def _second_run_cannot_read(_message: str) -> None: + sent["n"] += 1 + if sent["n"] == 2: + ws.read_error = "HTTP 502" + + turns = [ + _turn("Created.", {"metric_id": "m", "maql": "SELECT 2", "title": "M"}), + _turn("Created.", {"metric_id": "m", "maql": "SELECT 1", "title": "M"}), + ] + with _run_env(turns, ws, on_send=_second_run_cannot_read): + summary = run_agentic_metric_skill( + "http://h", "t", "ws", "q", {"candidates": ["SELECT 1"], "expect_suggestions": True}, k=2 + ) + graded, unscored = summary.run_results + assert graded.harness_error is None and unscored.harness_error is not None + assert summary.best is graded + + +def test_the_assignment_search_gives_up_instead_of_stalling() -> None: + same = [{"metric_id": f"m{i}", "maql": "SELECT 1"} for i in range(8)] + expected = (*(("SELECT 1",),) * 8, ("SELECT 2",)) + with pytest.raises(metric_skill_mod.MetricMatchUndecided, match="no verdict after"): + _match_expected_metrics(expected, same) + + +def test_an_undecided_match_is_a_harness_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(metric_skill_mod, "_MATCH_SEARCH_STEPS", 1) + ws = _FakeWorkspace({}) + both = [{"metric_id": "a", "maql": "SELECT 1"}, {"metric_id": "b", "maql": "SELECT 2"}] + with _run_env([_turn("Created both.", *both)], ws) as (client, _): + summary = _run({"expected_metrics": [["SELECT 1"], ["SELECT 2"]], "user_replies": ["Yes."]}) + assert client.send_message.call_count == 1 + assert summary.best.checks["metric_all_created"] is False + assert (summary.best.harness_error or "").startswith("metric_all_created: no verdict after 1 assignments") + + +def test_a_malformed_write_result_does_not_stop_the_cleanup_of_the_others() -> None: + ws = _FakeWorkspace({}) + turn = ChatResult.model_validate( + { + "textResponse": "Created.", + "toolCallEvents": [ + {"functionName": "create_metric", "functionArguments": "{}", "result": '["bad"]'}, + { + "functionName": "create_metric", + "functionArguments": "{}", + "result": json.dumps({"data": {"metric_id": "m", "maql": "SELECT 1", "created_new": True}}), + }, + ], + } + ) + with _run_env([turn], ws), patch("gooddata_eval.core.agentic.metric_skill._delete_metric") as delete: + summary = _run({"candidates": ["SELECT 1"]}) + assert summary.best.passed + assert [c.args[2] for c in delete.call_args_list] == ["m"]