Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if
from gooddata_eval.core.chat.sse_client import bind_run_retention
from gooddata_eval.core.config import ReasoningEffort
from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
from gooddata_eval.core.evaluators.visualization import expectation_options
Expand Down Expand Up @@ -662,7 +663,10 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
# regardless), so this bounds the wait at one wave rather than the dataset.
pool = ThreadPoolExecutor(max_workers=concurrency, thread_name_prefix="agentic")
try:
futures = {pool.submit(_process_item, i, it): i for i, it in parallel}
# Each evaluator builds its ChatClient inside the worker, so the run's
# --keep-conversations has to be carried into the thread explicitly.
process_item = bind_run_retention(_process_item)
futures = {pool.submit(process_item, i, it): i for i, it in parallel}
for future in as_completed(futures):
results[futures[future]] = future.result()
except BaseException:
Expand Down
25 changes: 22 additions & 3 deletions packages/gooddata-eval/src/gooddata_eval/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,12 @@

from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items
from gooddata_eval.core.catalog.http_client import CatalogMetadataClient
from gooddata_eval.core.chat.sse_client import ChatClient, set_default_item_timeout, set_default_turn_timeout
from gooddata_eval.core.chat.sse_client import (
ChatClient,
keep_conversations,
set_default_item_timeout,
set_default_turn_timeout,
)
from gooddata_eval.core.config import (
DEFAULT_GATE,
DEFAULT_JUDGE_MODEL,
Expand Down Expand Up @@ -170,7 +175,17 @@ def _build_parser() -> argparse.ArgumentParser:
"--preserve-failed",
action="store_true",
dest="preserve_failed",
help="Keep failed conversations on the server for post-mortem inspection.",
help="Keep failed conversations on the server for post-mortem inspection. Single-turn "
"kinds only; use --keep-conversations to cover the agentic ones.",
)
run.add_argument(
"--keep-conversations",
action="store_true",
dest="keep_conversations",
help="Keep every conversation on the server, passed or failed, so the AI Interaction "
"Intelligence endpoints can be queried after the run (or set "
"GOODDATA_EVAL_KEEP_CONVERSATIONS=1). Covers the agentic kinds, which --preserve-failed "
"does not. Leaves state behind: only for a diagnostic run.",
)
run.add_argument(
"--reasoning-effort",
Expand Down Expand Up @@ -759,14 +774,18 @@ def main(argv: list[str] | None = None) -> int:
quiet=args.quiet,
kind=args.kind,
preserve_failed=args.preserve_failed,
keep_conversations=args.keep_conversations,
reasoning_effort=args.reasoning_effort,
gate=normalize_gate(args.gate),
agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
turn_timeout_s=args.turn_timeout,
item_timeout_s=args.item_timeout,
tags=tuple(dict.fromkeys(args.tags)),
)
return _run(config)
# Scoped to this run: main() is re-entrant under test and as a library call, and a
# leaked True would leave the next run's conversations on the server unasked.
with keep_conversations(config.keep_conversations):
return _run(config)
except (
ConnectionError_,
ModelResolutionError,
Expand Down
136 changes: 130 additions & 6 deletions packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,17 @@
protocol, not on this class.
"""

import contextlib
import contextvars
import functools
import json
import logging
import os
import re
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any, Iterable
from typing import Any, Iterable, Iterator, ParamSpec, TypeVar
from urllib.parse import quote

import httpx
Expand All @@ -38,6 +41,9 @@

_log = logging.getLogger(__name__)

_P = ParamSpec("_P")
_R = TypeVar("_R")

# The retry helpers live in core/_retry.py, shared with the takeaway client. The private
# names stay importable from here: gdc-nas tavern-e2e imports them from this module.
_int_env = int_env
Expand Down Expand Up @@ -120,6 +126,16 @@ class TurnTimeoutError(ChatError):
retrying it spends the budget again."""


def _bool_env(name: str, default: bool) -> bool:
"""Read a flag from the environment, falling back to ``default`` when unset or blank.

``0``/``false``/``no``/``off`` are false, anything else present is true -- so the usual
``VAR=0`` turns a flag off rather than reading as a non-empty, therefore true, string.
"""
raw = os.getenv(name)
return raw.strip().lower() not in ("0", "false", "no", "off") if raw and raw.strip() else default


# Wall-clock cap on a single agent turn, 0 = uncapped. httpx's `timeout` is per-read,
# so an agent that keeps emitting reasoning events can stream for many minutes without
# ever tripping it -- this is what bounds a runaway item and lets the run move on.
Expand All @@ -140,6 +156,46 @@ class TurnTimeoutError(ChatError):
_TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
_TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+")

# When set, no conversation is deleted -- pass or fail, single-turn or agentic. The
# server-side record is what the Interaction Intelligence endpoints read, so a run meant to
# be inspected afterwards has to leave it behind. Enforced inside delete_conversation rather
# than at each call site: thirteen agentic evaluators call it twenty times between them, and
# a fourteenth kind would have been one more place to remember.
_KEEP_CONVERSATIONS = _bool_env("GOODDATA_EVAL_KEEP_CONVERSATIONS", False)

# A run's --keep-conversations, kept apart from the process default above. A ContextVar so
# that runs overlapping in one process cannot see or undo each other's override. It reaches a
# pool worker only through the context the runner copies into each task it submits.
_RUN_KEEP_CONVERSATIONS: contextvars.ContextVar[bool] = contextvars.ContextVar(
"gd_eval_run_keep_conversations", default=False
)


def _retention_on() -> bool:
"""Whether a conversation ending now should be kept: the run asked to, or the process does."""
return _RUN_KEEP_CONVERSATIONS.get() or _KEEP_CONVERSATIONS


def bind_run_retention(fn: Callable[_P, _R]) -> Callable[_P, _R]:
"""Wrap ``fn`` so it runs under the calling run's ``keep_conversations`` in any thread.

A pool worker does not inherit the submitting thread's ContextVars. Only this one is
carried, not the whole context: the runners' other per-item ContextVars rely on each
worker starting from its own defaults. Set and reset per call, so a pooled thread that
later serves another run does not keep this one's override.
"""
keep = _RUN_KEEP_CONVERSATIONS.get()

@functools.wraps(fn)
def _bound(*args: _P.args, **kwargs: _P.kwargs) -> _R:
token = _RUN_KEEP_CONVERSATIONS.set(keep)
try:
return fn(*args, **kwargs)
finally:
_RUN_KEEP_CONVERSATIONS.reset(token)

return _bound


@functools.cache
def _warn_skipped_trace_label(key: str) -> None:
Expand Down Expand Up @@ -186,6 +242,65 @@ def set_default_item_timeout(seconds: float | None) -> None:
_ITEM_TIMEOUT_S = seconds or 0.0


def set_keep_conversations(keep: bool) -> None:
"""Keep every conversation this process creates, whatever its outcome.

Lands here for the same reason the timeout defaults do -- the agentic evaluators build
their own clients deep in the call tree. A client keeps its conversations if retention
was on when it was built or is on when it deletes, so turning retention on reaches a
client already built, and turning it off does not reach one built while it was on.

This is the process-wide default, shared by every run in the process. For a single run
use ``keep_conversations``, which leaves the default alone: a bare set leaks into
whatever the process does next, and the direction of the leak is the costly one -- a
later run that asked for nothing would silently leave server-side state behind.

Args:
keep: The new process default. ``False`` turns retention off for clients built from
now on, whatever ``GOODDATA_EVAL_KEEP_CONVERSATIONS`` says, unless their run
asked for retention through ``keep_conversations``.
"""
global _KEEP_CONVERSATIONS
_KEEP_CONVERSATIONS = keep


@contextlib.contextmanager
def keep_conversations(keep: bool) -> Iterator[None]:
"""Turn conversation retention on for one run, without touching the process default.

The override lives in a ContextVar, so it applies to this context and to the contexts
the runners copy into their pool workers, and to nothing else. Runs overlapping in one
process each see only their own override, and leaving the scope resets this context
alone -- there is no shared setting to restore, so the last run to exit cannot leave
retention on for the runs after it.

A worker still running when the run exits -- one left behind by a Ctrl-C that cancelled
only the queued items -- runs in its own copied context, and its client also holds the
policy it was built under, so it cannot delete a conversation the run asked to keep.

Args:
keep: ``True`` turns retention on for the run. ``False`` is not "delete": it supplies
no override, so an operator who exported ``GOODDATA_EVAL_KEEP_CONVERSATIONS``
still gets retention on a run that passes no flag.
"""
if not keep:
yield
return
token = _RUN_KEEP_CONVERSATIONS.set(True)
try:
yield
finally:
_RUN_KEEP_CONVERSATIONS.reset(token)


def _attach_conversation_id(exc: BaseException, conversation_id: str) -> None:
"""Hang ``conversation_id`` on a raised exception, where the runner reads it for the report."""
try:
object.__setattr__(exc, "conversation_id", conversation_id)
except TypeError:
pass # C-extension exception that rejects __setattr__


def _is_retryable_exc(exc: Exception) -> bool:
if isinstance(exc, TransientChatError):
return True
Expand Down Expand Up @@ -528,6 +643,9 @@ def __init__(
http_timeout = httpx.Timeout(timeout, read=min(timeout, *caps))
self._client = httpx.Client(timeout=http_timeout)
self._preserve_failed = preserve_failed
# Captured so the policy of the run that built this client outlives that run's scope;
# see keep_conversations.
self._keep_conversations = _retention_on()
self._reasoning_effort = normalize_reasoning_effort(reasoning_effort)
self._document_format = normalize_document_format(document_format)
self._agent_id = agent_id
Expand Down Expand Up @@ -555,6 +673,8 @@ def _do() -> str:
return conversation_id

def delete_conversation(self, conversation_id: str) -> None:
if self._keep_conversations or _retention_on():
return
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
try:
self._client.delete(f"{self._base}/{conversation_id}", headers=self._auth)
except httpx.HTTPError:
Expand Down Expand Up @@ -631,7 +751,14 @@ def _do() -> ChatResult:

try:
result = retry_transient(_do, is_retryable=_is_retryable_exc)
except ChatError as exc:
except Exception as exc:
# The runner reads it off the exception to fill the report's conversation_id: a
# conversation kept by --keep-conversations or --preserve-failed is otherwise
# unfindable once its turn has raised. An evaluator that set its own keeps it.
if getattr(exc, "conversation_id", None) is None:
_attach_conversation_id(exc, conversation_id)
if not isinstance(exc, ChatError):
raise
if joined is not None and exc.partial_result is not None:
self._record_join(conversation_id, exc.partial_result.trace_id, joined)
# Only a turn the agent itself ran long or left unanswered has a response time worth
Expand Down Expand Up @@ -704,10 +831,7 @@ def ask(self, item: DatasetItem) -> ChatResult:
success = True
return result
except Exception as exc:
try:
object.__setattr__(exc, "conversation_id", conversation_id)
except TypeError:
pass # C-extension exception that rejects __setattr__
_attach_conversation_id(exc, conversation_id)
raise
finally:
if success or not self._preserve_failed:
Expand Down
3 changes: 3 additions & 0 deletions packages/gooddata-eval/src/gooddata_eval/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@ class RunConfig:
quiet: bool = False
kind: str = "visualization"
preserve_failed: bool = False
keep_conversations: bool = False
"""Keep every conversation, pass or fail. Superset of ``preserve_failed``; applies to the
agentic kinds too, which ``preserve_failed`` never reached."""
reasoning_effort: ReasoningEffort | None = None
agent_id: str | None = None
turn_timeout_s: float | None = None
Expand Down
65 changes: 65 additions & 0 deletions packages/gooddata-eval/tests/test_agentic_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise
import importlib
import inspect
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from unittest.mock import patch

Expand All @@ -20,6 +22,7 @@
runs_in_parallel,
)
from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError
from gooddata_eval.core.chat import sse_client as sse_mod
from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
from gooddata_eval.core.langfuse.item_scope import ItemTraceScope, current_scope
from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, CreatedVisualization, DatasetItem
Expand Down Expand Up @@ -1410,3 +1413,65 @@ def evaluate(*args: Any, **kwargs: Any) -> AgenticEvalOutcome:
assert {i.id: i.trace_latencies_s for i in report.items} == {
f"g{n}": [n * 100.0, n * 100.0 + 1, n * 100.0 + 2] for n in range(1, 5)
}


@pytest.mark.parametrize("keep", [True, False])
def test_parallel_agentic_workers_see_the_runs_keep_conversations(monkeypatch: pytest.MonkeyPatch, keep: bool) -> None:
"""The agentic evaluators build their ChatClient inside a pool worker, which does not
inherit the submitting thread's ContextVars; the runner has to carry the run's choice in."""
monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False)
items = [
DatasetItem(
id=f"g{n}", dataset_name="d", test_kind="agentic_general_question", question=f"q{n}", expected_output="x"
)
for n in range(1, 4)
]
seen: list[bool] = []

def evaluate(*args: Any, **kwargs: Any) -> AgenticEvalOutcome:
seen.append(sse_mod._retention_on())
return AgenticEvalOutcome(reasoning_steps=[], detail={})

with (
patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", side_effect=evaluate),
sse_mod.keep_conversations(keep),
):
run_agentic_items(
items, host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01", concurrency=3
)

assert seen == [keep] * 3


def _agentic_module_sources() -> dict[str, str]:
"""Every agentic evaluator module's source, keyed by name. Discovered by scanning the
package so a new kind is covered the day it lands rather than when someone remembers."""
package = importlib.import_module("gooddata_eval.core.agentic")
directory = Path(package.__file__).parent
return {
path.stem: path.read_text(encoding="utf-8")
for path in sorted(directory.glob("*.py"))
if not path.stem.startswith("_")
}


@pytest.mark.parametrize("module_name", sorted(_agentic_module_sources()))
def test_every_agentic_module_deletes_through_the_client_method(module_name: str) -> None:
"""--keep-conversations is enforced inside ChatClient.delete_conversation, so a module
that issues its own DELETE would quietly ignore it. Thirteen modules delete twenty times
between them; one bypass is enough to lose the conversation a diagnostic run needed."""
source = _agentic_module_sources()[module_name]
# Only HTTP deletes: `created.delete(sdk, workspace_id)` is workspace-object cleanup,
# which has nothing to do with conversation retention.
bypasses = [
line.strip() for line in source.splitlines() if re.search(r"\b(httpx|requests|_client)\.delete\(", line)
]
assert not bypasses, f"{module_name} issues its own DELETE; call client.delete_conversation instead: {bypasses}"


def test_the_agentic_modules_do_delete_conversations() -> None:
"""Guards the guard: an assertion that only forbids a pattern still passes once the
cleanup it is protecting has been deleted outright."""
sources = _agentic_module_sources()
deleting = {name for name, src in sources.items() if "delete_conversation(" in src}
assert len(deleting) >= 12, f"expected most agentic kinds to clean up, found {sorted(deleting)}"
Loading
Loading