From 85c8426e2bdff62e9ecc237b8d12f9dc0bea8c43 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 19 Nov 2026 09:26:24 +0100 Subject: [PATCH] feat(gooddata-eval): select Langfuse dataset items by id or metadata Langfuse compares runs only within one dataset, so every variant of a question (its locales, say) belongs in one dataset. Running a single variant was not possible: load_langfuse_dataset took no item filter, and DatasetItem dropped the item metadata that says which variant an item is. - DatasetItem.metadata carries the source item metadata through verbatim. - load_langfuse_dataset(item_ids=, metadata_filter=) narrows the fetched items; both filters together must both match. An unknown id raises DatasetItemNotFoundError instead of silently running fewer items. - New repeatable CLI flags --langfuse-item-id and --langfuse-item-metadata KEY=VALUE (ANDed, compared as strings). A repeated metadata key is rejected. Both are refused with a local dataset rather than ignored. - --run-metadata-json PATH merges a JSON object into the Langfuse run metadata, on both the agentic and the single-shot path. The SDK-derived model, provider_type and reasoning_effort still win. Without the new flags, loading and run metadata are unchanged. jira: trivial risk: low Co-Authored-By: Claude Opus 5.5 --- packages/gooddata-eval/README.md | 10 ++ .../src/gooddata_eval/cli/agentic_runner.py | 10 +- .../src/gooddata_eval/cli/main.py | 97 ++++++++++++- .../src/gooddata_eval/core/config.py | 8 +- .../core/dataset/langfuse_source.py | 70 +++++++++- .../src/gooddata_eval/core/langfuse/sink.py | 16 ++- .../src/gooddata_eval/core/models.py | 6 + .../tests/test_agentic_runner.py | 30 ++++ packages/gooddata-eval/tests/test_cli.py | 72 +++++++++- .../tests/test_cli_langfuse_flags.py | 129 ++++++++++++++++++ .../gooddata-eval/tests/test_langfuse_sink.py | 38 ++++++ .../tests/test_langfuse_source.py | 118 ++++++++++++++++ 12 files changed, 592 insertions(+), 12 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_cli_langfuse_flags.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 9749259d1..af937c209 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -117,8 +117,17 @@ gd-eval run \ |---|---| | `--dataset PATH` | Flat folder of JSON files — one question per file. | | `--langfuse-dataset NAME` | Pull items by name from a Langfuse dataset. Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` and `LANGFUSE_BASE_URL` (or the legacy `LANGFUSE_HOST`). | +| `--langfuse-item-id ITEM_ID` | Run only this item from the Langfuse dataset. Repeatable. An id the dataset does not have is an error, not a skip — asking for twelve items and evaluating nine, silently, reads downstream as a pass-rate change. Requires `--langfuse-dataset`. | +| `--langfuse-item-metadata KEY=VALUE` | Run only the items whose Langfuse item metadata carries this key and value, e.g. `--langfuse-item-metadata language=es`. Repeatable; several are ANDed, and values compare as strings. Combined with `--langfuse-item-id`, an item must satisfy both. Requires `--langfuse-dataset`. | | `--kind TEST_KIND` | Fallback `test_kind` for dataset items that do not embed one. Defaults to `visualization`; use e.g. `agentic_metric_skill` for multi-turn agentic evaluation. Items that declare their own `test_kind` ignore this. | +Both item filters are applied after the fetch — the Langfuse dataset-items endpoint selects +by dataset but not by item id or item metadata. The fetch costs the same either way; what +the filters save is the part that actually costs, evaluating an item against a live agent. +They exist so one published dataset can hold every variant of a question (its locales, say) +and still be dispatched a slice at a time, instead of being split into datasets that +Langfuse can then no longer compare against each other. + #### Model selection | Flag | Description | @@ -159,6 +168,7 @@ interleaves when K > 1, and per-item latencies rise, so they stop being clean si | Flag | Description | |---|---| | `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Names each experiment run `{dataset_name}_{timestamp}_{model}`, suffixed `_effort-{level}` when `--reasoning-effort` is set (so runs differing only by effort stay separate) and `_run{N}` per run when `--runs` > 1 — e.g. `general_question_2026-09-02-11-13_gpt-5.2_run0`. Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` and `LANGFUSE_BASE_URL` (or the legacy `LANGFUSE_HOST`). | +| `--run-metadata-json PATH` | A JSON object merged into the Langfuse **dataset-run** metadata. Per run, not per item — for what is constant across the run and otherwise only recoverable from the repo that launched it: the agent id and its personality text, the workspace actually evaluated, the memory snapshot in force. A payload that is not an object is rejected up front, since run metadata merges key by key and a list would be dropped after the run had already cost its agent calls. Merged under `--tag`, which wins on a `tags` key. | ##### Langfuse v4 diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index e596c6128..3bde76878 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -5,6 +5,7 @@ import sys import time +from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime from typing import Any, TypedDict @@ -482,6 +483,7 @@ def run_agentic_items( concurrency: int = 1, gate: EvalGate = DEFAULT_GATE, tags: tuple[str, ...] = (), + run_metadata: Mapping[str, Any] | None = None, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport. @@ -494,10 +496,16 @@ def run_agentic_items( automatic ones. They reach Langfuse whenever trace linking runs, with or without ``use_langfuse``. + ``run_metadata`` is merged into the Langfuse dataset-run metadata, once per run rather + than once per item. ``tags`` wins on a "tags" key, since that one is derived here. + Results are collected in dataset order regardless of completion order. """ langfuse = make_langfuse_client() if use_langfuse else None - run_metadata_extra: dict[str, Any] | None = {"tags": list(tags)} if tags else None + extra: dict[str, Any] = dict(run_metadata or {}) + if tags: + extra["tags"] = list(tags) + run_metadata_extra: dict[str, Any] | None = extra or None report = EvalReport(model=model_version, gate=normalize_gate(gate)) total = len(items) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 071e455f9..b35930e11 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -2,6 +2,7 @@ """`gd-eval` command-line entry point.""" import argparse +import json import os import sys import threading @@ -79,6 +80,35 @@ def _build_parser() -> argparse.ArgumentParser: source = run.add_mutually_exclusive_group(required=True) source.add_argument("--dataset", help="Path to a folder of dataset JSON files.") source.add_argument("--langfuse-dataset", dest="langfuse_dataset", help="Langfuse dataset name.") + run.add_argument( + "--langfuse-item-id", + dest="langfuse_item_ids", + action="append", + default=[], + metavar="ITEM_ID", + help="Run only this Langfuse dataset item. Repeat for several. An id the dataset does " + "not have is an error, not a skip. Requires --langfuse-dataset.", + ) + run.add_argument( + "--langfuse-item-metadata", + dest="langfuse_item_metadata", + action="append", + default=[], + type=_parse_item_metadata, + metavar="KEY=VALUE", + help="Run only Langfuse items whose metadata carries this key and value (e.g. " + "--langfuse-item-metadata language=es). Repeat to require several; values compare as " + "strings. Combined with --langfuse-item-id, an item must satisfy both. " + "Requires --langfuse-dataset.", + ) + run.add_argument( + "--run-metadata-json", + dest="run_metadata_json", + metavar="PATH", + help="JSON object merged into the Langfuse dataset-run metadata. Per run, not per item " + "-- use it for what is constant across the run (agent id, personality, workspace, " + "memory snapshot id), not for anything that varies by question.", + ) run.add_argument( "--kind", dest="kind", @@ -397,6 +427,44 @@ def _parse_tag(val: str) -> str: return tag +def _parse_item_metadata(val: str) -> tuple[str, str]: + """Split one --langfuse-item-metadata KEY=VALUE pair.""" + key, sep, value = val.partition("=") + if not sep or not key.strip(): + raise argparse.ArgumentTypeError(f"expected KEY=VALUE, got {val!r}") + return key.strip(), value.strip() + + +def _first_duplicate_metadata_key(pairs: list[tuple[str, str]]) -> str | None: + """Return the first key repeated across --langfuse-item-metadata pairs, else None. + + `dict(pairs)` silently keeps only the last value for a repeated key, which would read + as the filter the item satisfied rather than a flag mistake. + """ + seen: set[str] = set() + for key, _value in pairs: + if key in seen: + return key + seen.add(key) + return None + + +def _load_run_metadata(path: str) -> dict[str, Any]: + """Read --run-metadata-json. + + A non-object payload is rejected here rather than at the Langfuse call: dataset-run + metadata is merged key by key, so a list or a bare string has nowhere to go and would + otherwise be dropped silently after the run had already cost its agent calls. + """ + try: + payload = json.loads(Path(path).read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + raise ValueError(f"--run-metadata-json {path}: not valid JSON ({e})") from e + if not isinstance(payload, dict): + raise ValueError(f"--run-metadata-json {path}: expected a JSON object, got {type(payload).__name__}") + return payload + + def _make_progress_callbacks(console: Console): """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress. @@ -447,7 +515,12 @@ def _load_dataset(config: RunConfig): if config.langfuse_dataset is None: # pragma: no cover - argparse mutually-exclusive group guarantees one is set raise ValueError("Either --dataset or --langfuse-dataset is required.") - return load_langfuse_dataset(config.langfuse_dataset, default_test_kind=config.kind) + return load_langfuse_dataset( + config.langfuse_dataset, + default_test_kind=config.kind, + item_ids=config.item_ids or None, + metadata_filter=config.item_metadata_filter or None, + ) def _list_models(host: str, token: str, workspace_id: str | None) -> int: @@ -511,6 +584,15 @@ def _run(config: RunConfig) -> int: file=sys.stderr, ) return _EXIT_OPERATIONAL_ERROR + if (config.item_ids or config.item_metadata_filter) and config.langfuse_dataset is None: + # Refused rather than ignored: a run that quietly evaluated the whole local folder + # after being asked for one language would be read as that language's pass rate. + print( + "error: --langfuse-item-id / --langfuse-item-metadata require --langfuse-dataset " + "(a local dataset folder has no item metadata to select on).", + file=sys.stderr, + ) + return _EXIT_OPERATIONAL_ERROR items = _load_dataset(config) agentic_items = [i for i in items if i.test_kind in AGENTIC_TEST_KINDS] @@ -570,6 +652,7 @@ def _run(config: RunConfig) -> int: provider_type=resolved.provider_type, reasoning_effort=config.reasoning_effort, tags=config.tags, + run_metadata=config.run_metadata, ) def on_langfuse_item_done( @@ -596,6 +679,7 @@ def on_langfuse_item_done( reasoning_effort=config.reasoning_effort, use_langfuse=config.log_to_langfuse, tags=config.tags, + run_metadata=config.run_metadata, run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, @@ -731,6 +815,14 @@ def main(argv: list[str] | None = None) -> int: if hasattr(args, "concurrency") and args.concurrency < 1: print("error: --concurrency must be >= 1.", file=sys.stderr) return _EXIT_OPERATIONAL_ERROR + if hasattr(args, "langfuse_item_metadata"): + dup_key = _first_duplicate_metadata_key(args.langfuse_item_metadata) + if dup_key is not None: + print( + f"error: --langfuse-item-metadata repeats key {dup_key!r}; each key may be given once.", + file=sys.stderr, + ) + return _EXIT_OPERATIONAL_ERROR try: # Rendering existing JSON needs no host, token or workspace -- dispatch before # resolve_connection so `report` works on a laptop with no credentials at all. @@ -765,6 +857,9 @@ def main(argv: list[str] | None = None) -> int: turn_timeout_s=args.turn_timeout, item_timeout_s=args.item_timeout, tags=tuple(dict.fromkeys(args.tags)), + item_ids=tuple(dict.fromkeys(args.langfuse_item_ids)), + item_metadata_filter=dict(args.langfuse_item_metadata), + run_metadata=_load_run_metadata(args.run_metadata_json) if args.run_metadata_json else {}, ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index bb5f838ae..8cc23ce2d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -4,7 +4,7 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import Literal, cast, get_args +from typing import Any, Literal, cast, get_args # Values that mean "off" when read from the environment. Needed because every non-empty # string is truthy in Python, so a bare bool() on an env var reads FOO=0 -- the obvious way @@ -118,3 +118,9 @@ class RunConfig: gate: EvalGate = DEFAULT_GATE tags: tuple[str, ...] = () """Custom Langfuse trace tags, added after the automatic ones.""" + item_ids: tuple[str, ...] = () + """Langfuse dataset items to run, by id. Empty runs them all. Ignored for a local dataset.""" + item_metadata_filter: dict[str, str] = field(default_factory=dict) + """Run only Langfuse items whose metadata carries all these key/values. Ignored for a local dataset.""" + run_metadata: dict[str, Any] = field(default_factory=dict) + """Extra key/values written into the Langfuse dataset-run metadata, per run, not per item.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py index 878bac8e7..e80585446 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py @@ -12,6 +12,7 @@ LANGFUSE_HOST — base URL, legacy alias for LANGFUSE_BASE_URL """ +from collections.abc import Collection, Mapping from typing import Any, TypeVar, cast import httpx @@ -25,6 +26,19 @@ _T = TypeVar("_T") +class DatasetItemNotFoundError(LookupError): + """Named item ids were asked for and the dataset does not have them. + + Raised rather than silently returning the ids that did exist: asking for twelve items + and evaluating nine, with nothing said, reads downstream as a pass-rate change. + """ + + def __init__(self, dataset_name: str, missing: Collection[str]) -> None: + self.dataset_name = dataset_name + self.missing = sorted(missing) + super().__init__(f"Langfuse dataset {dataset_name!r} has no item(s) with id: {', '.join(self.missing)}") + + def _make_client() -> httpx.Client: """Build an httpx client with Langfuse basic-auth headers.""" return make_http_client(timeout=30) @@ -178,21 +192,52 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt takeaway_input=takeaway_input, catalog_input=catalog_input, user_context=_user_context_from_raw(raw), + metadata=metadata if isinstance(metadata := raw.get("metadata"), dict) else {}, ) -def load_langfuse_dataset(name: str, *, default_test_kind: str = "visualization") -> list[DatasetItem]: - """Pull all items from a Langfuse dataset by name via the REST API. +def _matches_metadata(item: DatasetItem, metadata_filter: Mapping[str, str]) -> bool: + """True when the item's metadata carries every wanted key with the wanted value. + + Values are compared as strings because the filter arrives from a CLI `KEY=VALUE` pair, + where everything is a string, while the item side may hold a number or a bool. A key + the item does not have never matches -- an item with no `language` is not an item in + every language. + """ + return all(key in item.metadata and str(item.metadata[key]) == value for key, value in metadata_filter.items()) + + +def load_langfuse_dataset( + name: str, + *, + default_test_kind: str = "visualization", + item_ids: Collection[str] | None = None, + metadata_filter: Mapping[str, str] | None = None, +) -> list[DatasetItem]: + """Pull items from a Langfuse dataset by name via the REST API. + + Both filters are applied client-side, after the fetch: the Langfuse dataset-items + endpoint can select by dataset but not by item id or item metadata, so narrowing has + to happen here. The fetch cost is the whole dataset either way; what the filters save + is the part that actually costs -- evaluating an item against a live agent. Args: name: The Langfuse dataset name (as shown in the Langfuse UI). default_test_kind: Fallback test_kind when the item doesn't specify one. + item_ids: Keep only these item ids. Every id must exist, or it raises -- see + `DatasetItemNotFoundError`. + metadata_filter: Keep only items whose metadata carries all of these key/value + pairs, compared as strings. + + Given both, an item has to satisfy both. Returns: - Parsed dataset items. + Parsed dataset items, in the order Langfuse returned them. Raises: RuntimeError: Missing Langfuse credentials or dataset not found. + ValueError: The dataset is empty, or the filters selected nothing from it. + DatasetItemNotFoundError: `item_ids` named an id the dataset does not have. """ items: list[dict] = [] page = 1 @@ -219,4 +264,21 @@ def load_langfuse_dataset(name: str, *, default_test_kind: str = "visualization" if not items: raise ValueError(f"Langfuse dataset '{name}' exists but contains no items.") - return [_item_from_raw(raw, dataset_name=name, test_kind=default_test_kind) for raw in items] + parsed = [_item_from_raw(raw, dataset_name=name, test_kind=default_test_kind) for raw in items] + + if item_ids is not None: + wanted = set(item_ids) + if missing := wanted - {item.id for item in parsed}: + raise DatasetItemNotFoundError(name, missing) + parsed = [item for item in parsed if item.id in wanted] + + if metadata_filter: + parsed = [item for item in parsed if _matches_metadata(item, metadata_filter)] + + if not parsed: + raise ValueError( + f"Langfuse dataset '{name}' has items, but none matched the requested filter " + f"(metadata_filter={dict(metadata_filter or {})!r})." + ) + + return parsed diff --git a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py index 1e3e6bc3f..9bcace942 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py @@ -65,6 +65,7 @@ def __init__( reasoning_effort: ReasoningEffort | None = None, *, tags: tuple[str, ...] = (), + run_metadata: dict[str, Any] | None = None, transport: httpx.BaseTransport | None = None, ): self._dataset_name = dataset_name @@ -73,6 +74,7 @@ def __init__( self._provider_type = provider_type self._reasoning_effort = reasoning_effort self._tags = tags + self._run_metadata = dict(run_metadata) if run_metadata else {} self._client = HttpxLangfuseClient(timeout=10.0, transport=transport) self._warned_unlinkable = False @@ -103,14 +105,20 @@ def _resolve_run(self, report: ItemReport, dataset_item_id: str) -> ExperimentRu if self._provider_type and self._model_id else self._model_id or None ) - return ExperimentRun( - self._run_name, - dataset_id, + # User-supplied run_metadata first, then the automatic fields -- so an automatic + # field cannot be overwritten by --run-metadata-json, matching the agentic path. + metadata: dict[str, Any] = dict(self._run_metadata) + metadata.update( { "model": self._model_id, "provider_type": self._provider_type, "reasoning_effort": self._reasoning_effort, - }, + } + ) + return ExperimentRun( + self._run_name, + dataset_id, + metadata, description=description, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index eceda9f54..988b0d3e5 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -599,3 +599,9 @@ class DatasetItem(BaseModel): # that schema (a discriminated union of view/widget descriptors), so re-modelling it here # would only create a second copy to keep in sync. user_context: dict[str, Any] | None = None + # The source's own item metadata, carried through verbatim and opaque to evaluators. A + # Langfuse dataset item can hold whatever its publisher put there -- language, complexity, + # workspace id, the git revision the fixture was published from -- and callers select on it + # and report it. Fields the loader derives from it (`test_kind`, `summary_input`, + # `user_context`) are exposed separately; this dict stays intact, keys included. + metadata: dict[str, Any] = Field(default_factory=dict) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 26abd44e2..c21c9cf7f 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1164,6 +1164,36 @@ def test_run_agentic_items_passes_custom_tags_as_run_metadata(tags, expected_ext assert mock_eval.call_args.kwargs["run_metadata_extra"] == expected_extra +@pytest.mark.parametrize( + ("run_metadata", "tags", "expected_extra"), + [ + # --run-metadata-json alone: reaches the dataset-run metadata with no tags key. + ({"agent_id": "a-1", "child_workspace_id": "ws-1"}, (), {"agent_id": "a-1", "child_workspace_id": "ws-1"}), + # Both: merged, not one replacing the other. + ({"agent_id": "a-1"}, ("nightly",), {"agent_id": "a-1", "tags": ["nightly"]}), + # A "tags" key in the JSON file loses -- tags is derived here from --tag. + ({"tags": ["from-json"]}, ("nightly",), {"tags": ["nightly"]}), + # Neither: forward None when no metadata or tags are supplied. + ({}, (), None), + ], +) +def test_run_agentic_items_merges_run_metadata_with_tags(run_metadata, tags, expected_extra): + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + return_value=AgenticEvalOutcome(detail={"alert_created": True}), + ) as mock_eval: + run_agentic_items( + [_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + tags=tags, + run_metadata=run_metadata, + ) + assert mock_eval.call_args.kwargs["run_metadata_extra"] == expected_extra + + class _FirstSend(BaseException): """Ends a run at its first chat turn; BaseException so no kind's error handling keeps it.""" diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 4265612ff..b8844b5c3 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -1,5 +1,6 @@ # (C) 2026 GoodData Corporation import io +import json from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime from unittest.mock import MagicMock @@ -393,7 +394,16 @@ def close(self): ... langfuse_calls: list = [] class _FakeSink: - def __init__(self, dataset_name, run_name, model_id="", provider_type="", reasoning_effort=None, tags=()): ... + def __init__( + self, + dataset_name, + run_name, + model_id="", + provider_type="", + reasoning_effort=None, + tags=(), + run_metadata=None, + ): ... def log_item(self, report, *, dataset_item_id): langfuse_calls.append(dataset_item_id) @@ -1026,6 +1036,66 @@ def _fake_run_agentic(items, **kwargs): assert agentic_kwargs["tags"] == ("acme", "nightly") +def test_cli_run_metadata_json_reaches_the_non_agentic_langfuse_sink(monkeypatch, tmp_path): + """--run-metadata-json used to only reach run_agentic_items; run_items got nothing.""" + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel(provider_id="prov", model_id="gpt-5.2", switched=False, provider_name="P") + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + monkeypatch.setattr(cli_main, "ChatClient", lambda **k: object()) + monkeypatch.setattr( + cli_main, + "_load_dataset", + lambda config: [ + DatasetItem(id="q1", dataset_name="d", test_kind="visualization", question="q", expected_output="a") + ], + ) + monkeypatch.setattr(cli_main, "run_items", lambda items, backend, **kw: EvalReport(model="gpt-5.2")) + + sink_kwargs: dict = {} + + class _FakeSink: + def __init__(self, **kwargs): + sink_kwargs.update(kwargs) + + def log_item(self, report, *, dataset_item_id): ... + + monkeypatch.setattr(cli_main, "LangfuseSink", _FakeSink) + + metadata_path = tmp_path / "run_metadata.json" + metadata_path.write_text(json.dumps({"agent_id": "a-1"}), encoding="utf-8") + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--langfuse-dataset", + "my_dataset", + "--langfuse", + "--run-metadata-json", + str(metadata_path), + ] + ) + + assert exit_code == 0 + assert sink_kwargs["run_metadata"] == {"agent_id": "a-1"} + + def test_cli_timers_flag_enables_timer_output(monkeypatch): # --timers is the discoverable front door for GD_EVAL_TIMERS. The env var is the # mechanism because the [timer] call sites sit four layers below the CLI, inside the diff --git a/packages/gooddata-eval/tests/test_cli_langfuse_flags.py b/packages/gooddata-eval/tests/test_cli_langfuse_flags.py new file mode 100644 index 000000000..0c0368eec --- /dev/null +++ b/packages/gooddata-eval/tests/test_cli_langfuse_flags.py @@ -0,0 +1,129 @@ +# (C) 2026 GoodData Corporation +"""--langfuse-item-id / --langfuse-item-metadata / --run-metadata-json. + +The three flags that let a caller run part of a Langfuse dataset and say what the run was, +plus a regression guard that a run using none of them is dispatched exactly as before. +""" + +import json +from unittest.mock import patch + +import pytest +from gooddata_eval.cli import main as cli_main +from gooddata_eval.core.config import RunConfig + +_BASE = ["run", "--host", "h", "--workspace", "w", "--langfuse-dataset", "mastercard_search_tool"] + + +def test_item_id_is_repeatable(): + args = cli_main.parse_args([*_BASE, "--langfuse-item-id", "q1-en", "--langfuse-item-id", "q2-en"]) + assert args.langfuse_item_ids == ["q1-en", "q2-en"] + + +def test_item_metadata_parses_key_value_pairs(): + args = cli_main.parse_args( + [*_BASE, "--langfuse-item-metadata", "language=es", "--langfuse-item-metadata", "complexity=Basic"] + ) + assert args.langfuse_item_metadata == [("language", "es"), ("complexity", "Basic")] + + +@pytest.mark.parametrize("bad", ["language", "=es", " =es"]) +def test_item_metadata_rejects_a_pair_without_a_key(bad): + with pytest.raises(SystemExit): + cli_main.parse_args([*_BASE, "--langfuse-item-metadata", bad]) + + +def test_item_metadata_keeps_an_empty_value(): + """`key=` is a real filter -- an item whose metadata holds "" for that key.""" + args = cli_main.parse_args([*_BASE, "--langfuse-item-metadata", "language="]) + assert args.langfuse_item_metadata == [("language", "")] + + +def test_first_duplicate_metadata_key_finds_a_repeat(): + assert cli_main._first_duplicate_metadata_key([("language", "en"), ("complexity", "Basic")]) is None + assert cli_main._first_duplicate_metadata_key([("language", "en"), ("language", "fr")]) == "language" + + +def test_cli_rejects_repeated_item_metadata_keys(capsys): + """dict(pairs) would silently keep only 'fr', reading as the filter the run asked for.""" + exit_code = cli_main.main( + [*_BASE, "--langfuse-item-metadata", "language=en", "--langfuse-item-metadata", "language=fr"] + ) + assert exit_code == 2 + assert "language" in capsys.readouterr().err + + +def test_all_three_flags_default_to_empty(): + args = cli_main.parse_args([*_BASE]) + assert args.langfuse_item_ids == [] + assert args.langfuse_item_metadata == [] + assert args.run_metadata_json is None + + +def test_run_metadata_json_is_read_from_the_file(tmp_path): + path = tmp_path / "run.json" + path.write_text(json.dumps({"agent_id": "a-1", "child_workspace_id": "ws-1"}), encoding="utf-8") + assert cli_main._load_run_metadata(str(path)) == {"agent_id": "a-1", "child_workspace_id": "ws-1"} + + +@pytest.mark.parametrize("payload", ["[1, 2]", '"a string"', "42"]) +def test_run_metadata_json_rejects_a_non_object(tmp_path, payload): + """Dataset-run metadata is merged key by key, so a list has nowhere to go.""" + path = tmp_path / "run.json" + path.write_text(payload, encoding="utf-8") + with pytest.raises(ValueError, match="expected a JSON object"): + cli_main._load_run_metadata(str(path)) + + +def test_run_metadata_json_rejects_malformed_json(tmp_path): + path = tmp_path / "run.json" + path.write_text("{not json", encoding="utf-8") + with pytest.raises(ValueError, match="not valid JSON"): + cli_main._load_run_metadata(str(path)) + + +def test_item_filters_reach_load_langfuse_dataset(): + config = RunConfig( + host="h", + token="t", + workspace_id="w", + langfuse_dataset="mastercard_search_tool", + kind="search_tool", + item_ids=("q1-en",), + item_metadata_filter={"language": "en"}, + ) + with patch("gooddata_eval.core.dataset.langfuse_source.load_langfuse_dataset") as load: + cli_main._load_dataset(config) + load.assert_called_once_with( + "mastercard_search_tool", + default_test_kind="search_tool", + item_ids=("q1-en",), + metadata_filter={"language": "en"}, + ) + + +def test_without_the_flags_the_loader_is_called_as_before(): + """Regression guard: an unfiltered Langfuse run must pass None, not an empty container. + + An empty tuple reaching `item_ids` would select zero items and raise, turning every + existing unfiltered run into an error. + """ + config = RunConfig(host="h", token="t", workspace_id="w", langfuse_dataset="ds", kind="search_tool") + with patch("gooddata_eval.core.dataset.langfuse_source.load_langfuse_dataset") as load: + cli_main._load_dataset(config) + load.assert_called_once_with("ds", default_test_kind="search_tool", item_ids=None, metadata_filter=None) + + +@pytest.mark.parametrize( + "extra", + [{"item_ids": ("q1-en",)}, {"item_metadata_filter": {"language": "es"}}], +) +def test_item_filters_with_a_local_dataset_are_refused(tmp_path, capsys, extra): + """Ignoring them would run the whole local folder and report it as that language.""" + config = RunConfig(host="h", token="t", workspace_id="w", dataset_folder=tmp_path, **extra) + assert cli_main._run(config) != 0 + assert "require --langfuse-dataset" in capsys.readouterr().err + + +# run_metadata's journey into the dataset-run metadata is covered behaviourally in +# test_agentic_runner.py, next to the --tag case it merges with. diff --git a/packages/gooddata-eval/tests/test_langfuse_sink.py b/packages/gooddata-eval/tests/test_langfuse_sink.py index 986f4ec4a..fe3468df6 100644 --- a/packages/gooddata-eval/tests/test_langfuse_sink.py +++ b/packages/gooddata-eval/tests/test_langfuse_sink.py @@ -265,6 +265,44 @@ def test_langfuse_sink_tags_without_custom_tags(monkeypatch): assert _tags_from(requests) == ["gd-eval", "visualization", "OPENAI", "effort-low"] +def _experiment_metadata_from(requests: list[httpx.Request]) -> dict[str, str]: + attrs = {a["key"]: a["value"] for a in _span_from(requests)["attributes"]} + prefix = "langfuse.experiment.metadata." + return {k[len(prefix) :]: v["stringValue"] for k, v in attrs.items() if k.startswith(prefix)} + + +def test_langfuse_sink_merges_run_metadata_into_experiment_metadata(monkeypatch): + requests: list[httpx.Request] = [] + sink = _make_sink( + monkeypatch, + _known_item_handler(requests), + model_id="gpt-5.4-mini", + run_metadata={"agent_id": "a-1", "child_workspace_id": "ws-1"}, + ) + + sink.log_item(_passing_report(), dataset_item_id="item-1") + + metadata = _experiment_metadata_from(requests) + assert metadata["agent_id"] == "a-1" + assert metadata["child_workspace_id"] == "ws-1" + assert metadata["model"] == "gpt-5.4-mini" + + +def test_langfuse_sink_run_metadata_cannot_override_automatic_fields(monkeypatch): + """Matches the agentic path: the SDK-derived model_version cannot be overwritten.""" + requests: list[httpx.Request] = [] + sink = _make_sink( + monkeypatch, + _known_item_handler(requests), + model_id="gpt-5.4-mini", + run_metadata={"model": "attacker-supplied-model"}, + ) + + sink.log_item(_passing_report(), dataset_item_id="item-1") + + assert _experiment_metadata_from(requests)["model"] == "gpt-5.4-mini" + + def test_langfuse_sink_appends_custom_tags_after_the_automatic_ones(monkeypatch): requests: list[httpx.Request] = [] sink = _make_sink( diff --git a/packages/gooddata-eval/tests/test_langfuse_source.py b/packages/gooddata-eval/tests/test_langfuse_source.py index 2a4ded41f..8dbe42607 100644 --- a/packages/gooddata-eval/tests/test_langfuse_source.py +++ b/packages/gooddata-eval/tests/test_langfuse_source.py @@ -3,6 +3,7 @@ import pytest from gooddata_eval.core.dataset.langfuse_source import ( + DatasetItemNotFoundError, _infer_test_kind, _item_from_raw, _make_client, @@ -309,6 +310,123 @@ def test_a_blank_test_kind_declaration_falls_back_to_the_default(): assert _infer_test_kind({"test_kind": " agentic_guardrail "}, "visualization") == "agentic_guardrail" +_EXPECTED = {"visualization": {"id": "x", "type": "", "query": {"fields": {}}}} + +# One question, four locales -- the shape every published fixture dataset actually has. +_LOCALE_ITEMS = [ + {**_raw_item(f"q1-{lang}", "Show revenue", _EXPECTED), "metadata": {"language": lang, "complexity": "Basic"}} + for lang in ("en", "es", "fr", "pt") +] + + +@pytest.fixture +def locale_dataset(monkeypatch): + """Serve `_LOCALE_ITEMS` from a mocked REST API, and hand back the mock client.""" + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"data": _LOCALE_ITEMS, "meta": {"totalItems": len(_LOCALE_ITEMS)}} + + mock_client = MagicMock() + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=False) + mock_client.get.return_value = mock_resp + + with patch("gooddata_eval.core.dataset.langfuse_source.httpx.Client", return_value=mock_client): + yield mock_client + + +def test_item_metadata_survives_the_load(locale_dataset): + """DatasetItem used to be `extra="ignore"` with no metadata field, so this was dropped.""" + items = load_langfuse_dataset("mastercard_search_tool") + assert items[0].metadata == {"language": "en", "complexity": "Basic"} + + +def test_an_item_without_metadata_gets_an_empty_dict(monkeypatch): + item = _item_from_raw(_raw_item("lf-nometa", "Show revenue", _EXPECTED), dataset_name="ds", test_kind="vis_agentic") + assert item.metadata == {} + + +def test_no_filter_returns_every_item(locale_dataset): + assert len(load_langfuse_dataset("mastercard_search_tool")) == 4 + + +def test_metadata_filter_selects_one_locale(locale_dataset): + items = load_langfuse_dataset("mastercard_search_tool", metadata_filter={"language": "es"}) + assert [item.id for item in items] == ["q1-es"] + + +def test_metadata_filter_requires_every_key_to_match(locale_dataset): + """Keys are ANDed; a dataset-wide key plus a selective one still narrows.""" + items = load_langfuse_dataset("mastercard_search_tool", metadata_filter={"language": "fr", "complexity": "Basic"}) + assert [item.id for item in items] == ["q1-fr"] + + +def test_metadata_filter_matching_nothing_raises_rather_than_running_zero_items(locale_dataset): + with pytest.raises(ValueError, match="none matched the requested filter"): + load_langfuse_dataset("mastercard_search_tool", metadata_filter={"language": "de"}) + + +def test_item_ids_selects_a_subset_and_keeps_source_order(locale_dataset): + items = load_langfuse_dataset("mastercard_search_tool", item_ids=["q1-pt", "q1-en"]) + assert [item.id for item in items] == ["q1-en", "q1-pt"] + + +def test_item_ids_and_metadata_filter_intersect(locale_dataset): + items = load_langfuse_dataset( + "mastercard_search_tool", item_ids=["q1-en", "q1-es"], metadata_filter={"language": "es"} + ) + assert [item.id for item in items] == ["q1-es"] + + +def test_an_unknown_item_id_raises_and_names_what_is_missing(locale_dataset): + """Silently returning the ids that did exist would read downstream as a pass-rate change.""" + with pytest.raises(DatasetItemNotFoundError) as excinfo: + load_langfuse_dataset("mastercard_search_tool", item_ids=["q1-en", "q1-de", "q1-zz"]) + assert excinfo.value.missing == ["q1-de", "q1-zz"] + assert "q1-de, q1-zz" in str(excinfo.value) + + +def test_a_filter_key_the_item_does_not_have_never_matches(locale_dataset): + """An item with no `runner` is not an item for every runner.""" + with pytest.raises(ValueError, match="none matched the requested filter"): + load_langfuse_dataset("mastercard_search_tool", metadata_filter={"runner": "1"}) + + +def test_metadata_values_compare_as_strings(monkeypatch): + """The filter arrives from a CLI `KEY=VALUE`, so the item's own int/bool must still match.""" + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-test") + monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + + raw = [{**_raw_item("q-num", "Show revenue", _EXPECTED), "metadata": {"runner": 1, "critical": False}}] + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"data": raw, "meta": {"totalItems": 1}} + mock_client = MagicMock() + mock_client.__enter__ = MagicMock(return_value=mock_client) + mock_client.__exit__ = MagicMock(return_value=False) + mock_client.get.return_value = mock_resp + + with patch("gooddata_eval.core.dataset.langfuse_source.httpx.Client", return_value=mock_client): + items = load_langfuse_dataset("ds", metadata_filter={"runner": "1", "critical": "False"}) + assert [item.id for item in items] == ["q-num"] + + +def test_filtering_does_not_change_the_request_sent_to_langfuse(locale_dataset): + """The endpoint cannot select by item id or metadata; filtering is ours, after the fetch. + + Guards against a later "optimisation" that passes these through as query params, which + Langfuse would ignore -- the filter would silently stop working. + """ + load_langfuse_dataset("mastercard_search_tool", item_ids=["q1-en"], metadata_filter={"language": "en"}) + params = locale_dataset.get.call_args[1]["params"] + assert set(params) == {"datasetName", "limit", "page"} + + def test_a_blank_declaration_does_not_hide_a_real_one_behind_it(): # The lookup must not stop at the first *string* it finds: a blank # expectedOutput.test_kind used to shadow a valid metadata.test_kind, and the item