diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index e596c6128..c4582b2dc 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -26,6 +26,7 @@ from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization from gooddata_eval.core.agentic.what_if import evaluate_agentic_what_if +from gooddata_eval.core.chat.sse_client import bind_run_retention from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.evaluators.visualization import expectation_options @@ -662,7 +663,10 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: # regardless), so this bounds the wait at one wave rather than the dataset. pool = ThreadPoolExecutor(max_workers=concurrency, thread_name_prefix="agentic") try: - futures = {pool.submit(_process_item, i, it): i for i, it in parallel} + # Each evaluator builds its ChatClient inside the worker, so the run's + # --keep-conversations has to be carried into the thread explicitly. + process_item = bind_run_retention(_process_item) + futures = {pool.submit(process_item, i, it): i for i, it in parallel} for future in as_completed(futures): results[futures[future]] = future.result() except BaseException: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 071e455f9..966522811 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -16,7 +16,12 @@ from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items from gooddata_eval.core.catalog.http_client import CatalogMetadataClient -from gooddata_eval.core.chat.sse_client import ChatClient, set_default_item_timeout, set_default_turn_timeout +from gooddata_eval.core.chat.sse_client import ( + ChatClient, + keep_conversations, + set_default_item_timeout, + set_default_turn_timeout, +) from gooddata_eval.core.config import ( DEFAULT_GATE, DEFAULT_JUDGE_MODEL, @@ -170,7 +175,17 @@ def _build_parser() -> argparse.ArgumentParser: "--preserve-failed", action="store_true", dest="preserve_failed", - help="Keep failed conversations on the server for post-mortem inspection.", + help="Keep failed conversations on the server for post-mortem inspection. Single-turn " + "kinds only; use --keep-conversations to cover the agentic ones.", + ) + run.add_argument( + "--keep-conversations", + action="store_true", + dest="keep_conversations", + help="Keep every conversation on the server, passed or failed, so the AI Interaction " + "Intelligence endpoints can be queried after the run (or set " + "GOODDATA_EVAL_KEEP_CONVERSATIONS=1). Covers the agentic kinds, which --preserve-failed " + "does not. Leaves state behind: only for a diagnostic run.", ) run.add_argument( "--reasoning-effort", @@ -759,6 +774,7 @@ def main(argv: list[str] | None = None) -> int: quiet=args.quiet, kind=args.kind, preserve_failed=args.preserve_failed, + keep_conversations=args.keep_conversations, reasoning_effort=args.reasoning_effort, gate=normalize_gate(args.gate), agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), @@ -766,7 +782,10 @@ def main(argv: list[str] | None = None) -> int: item_timeout_s=args.item_timeout, tags=tuple(dict.fromkeys(args.tags)), ) - return _run(config) + # Scoped to this run: main() is re-entrant under test and as a library call, and a + # leaked True would leave the next run's conversations on the server unasked. + with keep_conversations(config.keep_conversations): + return _run(config) except ( ConnectionError_, ModelResolutionError, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 78a7388aa..76f5f4809 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -13,14 +13,17 @@ protocol, not on this class. """ +import contextlib +import contextvars import functools import json import logging import os import re import time +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any, Iterable +from typing import Any, Iterable, Iterator, ParamSpec, TypeVar from urllib.parse import quote import httpx @@ -38,6 +41,9 @@ _log = logging.getLogger(__name__) +_P = ParamSpec("_P") +_R = TypeVar("_R") + # The retry helpers live in core/_retry.py, shared with the takeaway client. The private # names stay importable from here: gdc-nas tavern-e2e imports them from this module. _int_env = int_env @@ -120,6 +126,16 @@ class TurnTimeoutError(ChatError): retrying it spends the budget again.""" +def _bool_env(name: str, default: bool) -> bool: + """Read a flag from the environment, falling back to ``default`` when unset or blank. + + ``0``/``false``/``no``/``off`` are false, anything else present is true -- so the usual + ``VAR=0`` turns a flag off rather than reading as a non-empty, therefore true, string. + """ + raw = os.getenv(name) + return raw.strip().lower() not in ("0", "false", "no", "off") if raw and raw.strip() else default + + # Wall-clock cap on a single agent turn, 0 = uncapped. httpx's `timeout` is per-read, # so an agent that keeps emitting reasoning events can stream for many minutes without # ever tripping it -- this is what bounds a runaway item and lets the run move on. @@ -140,6 +156,46 @@ class TurnTimeoutError(ChatError): _TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS" _TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+") +# When set, no conversation is deleted -- pass or fail, single-turn or agentic. The +# server-side record is what the Interaction Intelligence endpoints read, so a run meant to +# be inspected afterwards has to leave it behind. Enforced inside delete_conversation rather +# than at each call site: thirteen agentic evaluators call it twenty times between them, and +# a fourteenth kind would have been one more place to remember. +_KEEP_CONVERSATIONS = _bool_env("GOODDATA_EVAL_KEEP_CONVERSATIONS", False) + +# A run's --keep-conversations, kept apart from the process default above. A ContextVar so +# that runs overlapping in one process cannot see or undo each other's override. It reaches a +# pool worker only through the context the runner copies into each task it submits. +_RUN_KEEP_CONVERSATIONS: contextvars.ContextVar[bool] = contextvars.ContextVar( + "gd_eval_run_keep_conversations", default=False +) + + +def _retention_on() -> bool: + """Whether a conversation ending now should be kept: the run asked to, or the process does.""" + return _RUN_KEEP_CONVERSATIONS.get() or _KEEP_CONVERSATIONS + + +def bind_run_retention(fn: Callable[_P, _R]) -> Callable[_P, _R]: + """Wrap ``fn`` so it runs under the calling run's ``keep_conversations`` in any thread. + + A pool worker does not inherit the submitting thread's ContextVars. Only this one is + carried, not the whole context: the runners' other per-item ContextVars rely on each + worker starting from its own defaults. Set and reset per call, so a pooled thread that + later serves another run does not keep this one's override. + """ + keep = _RUN_KEEP_CONVERSATIONS.get() + + @functools.wraps(fn) + def _bound(*args: _P.args, **kwargs: _P.kwargs) -> _R: + token = _RUN_KEEP_CONVERSATIONS.set(keep) + try: + return fn(*args, **kwargs) + finally: + _RUN_KEEP_CONVERSATIONS.reset(token) + + return _bound + @functools.cache def _warn_skipped_trace_label(key: str) -> None: @@ -186,6 +242,65 @@ def set_default_item_timeout(seconds: float | None) -> None: _ITEM_TIMEOUT_S = seconds or 0.0 +def set_keep_conversations(keep: bool) -> None: + """Keep every conversation this process creates, whatever its outcome. + + Lands here for the same reason the timeout defaults do -- the agentic evaluators build + their own clients deep in the call tree. A client keeps its conversations if retention + was on when it was built or is on when it deletes, so turning retention on reaches a + client already built, and turning it off does not reach one built while it was on. + + This is the process-wide default, shared by every run in the process. For a single run + use ``keep_conversations``, which leaves the default alone: a bare set leaks into + whatever the process does next, and the direction of the leak is the costly one -- a + later run that asked for nothing would silently leave server-side state behind. + + Args: + keep: The new process default. ``False`` turns retention off for clients built from + now on, whatever ``GOODDATA_EVAL_KEEP_CONVERSATIONS`` says, unless their run + asked for retention through ``keep_conversations``. + """ + global _KEEP_CONVERSATIONS + _KEEP_CONVERSATIONS = keep + + +@contextlib.contextmanager +def keep_conversations(keep: bool) -> Iterator[None]: + """Turn conversation retention on for one run, without touching the process default. + + The override lives in a ContextVar, so it applies to this context and to the contexts + the runners copy into their pool workers, and to nothing else. Runs overlapping in one + process each see only their own override, and leaving the scope resets this context + alone -- there is no shared setting to restore, so the last run to exit cannot leave + retention on for the runs after it. + + A worker still running when the run exits -- one left behind by a Ctrl-C that cancelled + only the queued items -- runs in its own copied context, and its client also holds the + policy it was built under, so it cannot delete a conversation the run asked to keep. + + Args: + keep: ``True`` turns retention on for the run. ``False`` is not "delete": it supplies + no override, so an operator who exported ``GOODDATA_EVAL_KEEP_CONVERSATIONS`` + still gets retention on a run that passes no flag. + """ + if not keep: + yield + return + token = _RUN_KEEP_CONVERSATIONS.set(True) + try: + yield + finally: + _RUN_KEEP_CONVERSATIONS.reset(token) + + +def _attach_conversation_id(exc: BaseException, conversation_id: str) -> None: + """Hang ``conversation_id`` on a raised exception, where the runner reads it for the report.""" + try: + object.__setattr__(exc, "conversation_id", conversation_id) + except TypeError: + pass # C-extension exception that rejects __setattr__ + + def _is_retryable_exc(exc: Exception) -> bool: if isinstance(exc, TransientChatError): return True @@ -528,6 +643,9 @@ def __init__( http_timeout = httpx.Timeout(timeout, read=min(timeout, *caps)) self._client = httpx.Client(timeout=http_timeout) self._preserve_failed = preserve_failed + # Captured so the policy of the run that built this client outlives that run's scope; + # see keep_conversations. + self._keep_conversations = _retention_on() self._reasoning_effort = normalize_reasoning_effort(reasoning_effort) self._document_format = normalize_document_format(document_format) self._agent_id = agent_id @@ -555,6 +673,8 @@ def _do() -> str: return conversation_id def delete_conversation(self, conversation_id: str) -> None: + if self._keep_conversations or _retention_on(): + return try: self._client.delete(f"{self._base}/{conversation_id}", headers=self._auth) except httpx.HTTPError: @@ -631,7 +751,14 @@ def _do() -> ChatResult: try: result = retry_transient(_do, is_retryable=_is_retryable_exc) - except ChatError as exc: + except Exception as exc: + # The runner reads it off the exception to fill the report's conversation_id: a + # conversation kept by --keep-conversations or --preserve-failed is otherwise + # unfindable once its turn has raised. An evaluator that set its own keeps it. + if getattr(exc, "conversation_id", None) is None: + _attach_conversation_id(exc, conversation_id) + if not isinstance(exc, ChatError): + raise if joined is not None and exc.partial_result is not None: self._record_join(conversation_id, exc.partial_result.trace_id, joined) # Only a turn the agent itself ran long or left unanswered has a response time worth @@ -704,10 +831,7 @@ def ask(self, item: DatasetItem) -> ChatResult: success = True return result except Exception as exc: - try: - object.__setattr__(exc, "conversation_id", conversation_id) - except TypeError: - pass # C-extension exception that rejects __setattr__ + _attach_conversation_id(exc, conversation_id) raise finally: if success or not self._preserve_failed: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index bb5f838ae..d0dd11bc1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -109,6 +109,9 @@ class RunConfig: quiet: bool = False kind: str = "visualization" preserve_failed: bool = False + keep_conversations: bool = False + """Keep every conversation, pass or fail. Superset of ``preserve_failed``; applies to the + agentic kinds too, which ``preserve_failed`` never reached.""" reasoning_effort: ReasoningEffort | None = None agent_id: str | None = None turn_timeout_s: float | None = None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 26abd44e2..7bccad776 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise import importlib import inspect +import re import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +from pathlib import Path from typing import Any from unittest.mock import patch @@ -20,6 +22,7 @@ runs_in_parallel, ) from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError +from gooddata_eval.core.chat import sse_client as sse_mod from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.langfuse.item_scope import ItemTraceScope, current_scope from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, CreatedVisualization, DatasetItem @@ -1410,3 +1413,65 @@ def evaluate(*args: Any, **kwargs: Any) -> AgenticEvalOutcome: assert {i.id: i.trace_latencies_s for i in report.items} == { f"g{n}": [n * 100.0, n * 100.0 + 1, n * 100.0 + 2] for n in range(1, 5) } + + +@pytest.mark.parametrize("keep", [True, False]) +def test_parallel_agentic_workers_see_the_runs_keep_conversations(monkeypatch: pytest.MonkeyPatch, keep: bool) -> None: + """The agentic evaluators build their ChatClient inside a pool worker, which does not + inherit the submitting thread's ContextVars; the runner has to carry the run's choice in.""" + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) + items = [ + DatasetItem( + id=f"g{n}", dataset_name="d", test_kind="agentic_general_question", question=f"q{n}", expected_output="x" + ) + for n in range(1, 4) + ] + seen: list[bool] = [] + + def evaluate(*args: Any, **kwargs: Any) -> AgenticEvalOutcome: + seen.append(sse_mod._retention_on()) + return AgenticEvalOutcome(reasoning_steps=[], detail={}) + + with ( + patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", side_effect=evaluate), + sse_mod.keep_conversations(keep), + ): + run_agentic_items( + items, host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01", concurrency=3 + ) + + assert seen == [keep] * 3 + + +def _agentic_module_sources() -> dict[str, str]: + """Every agentic evaluator module's source, keyed by name. Discovered by scanning the + package so a new kind is covered the day it lands rather than when someone remembers.""" + package = importlib.import_module("gooddata_eval.core.agentic") + directory = Path(package.__file__).parent + return { + path.stem: path.read_text(encoding="utf-8") + for path in sorted(directory.glob("*.py")) + if not path.stem.startswith("_") + } + + +@pytest.mark.parametrize("module_name", sorted(_agentic_module_sources())) +def test_every_agentic_module_deletes_through_the_client_method(module_name: str) -> None: + """--keep-conversations is enforced inside ChatClient.delete_conversation, so a module + that issues its own DELETE would quietly ignore it. Thirteen modules delete twenty times + between them; one bypass is enough to lose the conversation a diagnostic run needed.""" + source = _agentic_module_sources()[module_name] + # Only HTTP deletes: `created.delete(sdk, workspace_id)` is workspace-object cleanup, + # which has nothing to do with conversation retention. + bypasses = [ + line.strip() for line in source.splitlines() if re.search(r"\b(httpx|requests|_client)\.delete\(", line) + ] + assert not bypasses, f"{module_name} issues its own DELETE; call client.delete_conversation instead: {bypasses}" + + +def test_the_agentic_modules_do_delete_conversations() -> None: + """Guards the guard: an assertion that only forbids a pattern still passes once the + cleanup it is protecting has been deleted outright.""" + sources = _agentic_module_sources() + deleting = {name for name, src in sources.items() if "delete_conversation(" in src} + assert len(deleting) >= 12, f"expected most agentic kinds to clean up, found {sorted(deleting)}" diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 4265612ff..a2498b622 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -2,6 +2,8 @@ import io from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime +from pathlib import Path +from typing import Any from unittest.mock import MagicMock import httpx @@ -9,6 +11,7 @@ import pytest from gooddata_eval.cli import main as cli_main from gooddata_eval.cli.main import _parse_model_arg +from gooddata_eval.core.chat import sse_client as sse_mod from gooddata_eval.core.config import JUDGE_MODEL_ENV_VAR, RunConfig, judge_model from gooddata_eval.core.connection import ( ConnectionError_, # noqa: F401 - used in test_cli_operational_error_exits_nonzero @@ -704,6 +707,73 @@ def _fake_run(items, backend, *, runs, model, workspace_id, **kw): assert captured_kwargs.get("preserve_failed") is True +@pytest.mark.parametrize("env_default", [False, True]) +def test_cli_keep_conversations_flag_applies_process_wide( + monkeypatch: pytest.MonkeyPatch, fixtures_dir: Path, env_default: bool +) -> None: + """--keep-conversations lands on the module-level switch, not on the ChatClient kwargs. + + The agentic evaluators build their own clients deep in the call tree, so a constructor + kwarg would reach the single-turn path only -- which is the gap --preserve-failed has. + The setting is read while the run is active, so a scope that exited before ``_run`` fails. + """ + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + # Stands in for GOODDATA_EVAL_KEEP_CONVERSATIONS, which sets the module default at import. + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", env_default) + active_during_run: list[bool] = [] + + class _FakeController: + def __init__(self, *a: Any, **k: Any) -> None: ... + def get_active(self) -> ActiveLlmProvider: + return ActiveLlmProvider(provider_id="p", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested: str | None, provider: str | None = None) -> ResolvedModel: + return ResolvedModel(provider_id="p", model_id="gpt-5.2", switched=False, provider_name="P") + + def restore(self, original: ActiveLlmProvider) -> None: ... + def close(self) -> None: ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + monkeypatch.setattr(cli_main, "ChatClient", lambda **kwargs: object()) + + def _fake_run( + items: list[DatasetItem], backend: object, *, runs: int, model: str, workspace_id: str, **kw: Any + ) -> EvalReport: + active_during_run.append(sse_mod._retention_on()) + return EvalReport( + model=model, + workspace_id=workspace_id, + items=[ + ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=1) + ], + ) + + monkeypatch.setattr(cli_main, "run_items", _fake_run) + + argv = [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--quiet", + ] + assert cli_main.main([*argv, "--keep-conversations"]) == 0 + assert active_during_run == [True] + assert sse_mod._KEEP_CONVERSATIONS is env_default + + # Scoped per run, so a bare `run` afterwards falls back to the default: off unless the + # env var turned it on, which a run without the flag must not force off. + active_during_run.clear() + assert cli_main.main(argv) == 0 + assert active_during_run == [env_default] + assert sse_mod._KEEP_CONVERSATIONS is env_default + + def test_cli_rejects_negative_concurrency(monkeypatch, fixtures_dir): monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) exit_code = cli_main.main( diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 6f28a767a..f82266bf2 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -1,7 +1,9 @@ # (C) 2026 GoodData Corporation import json import logging +import threading from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor from typing import Any from urllib.parse import unquote @@ -866,6 +868,8 @@ def test_parse_sse_lines_first_response_id_wins(): def test_ask_attaches_conversation_id_to_result(monkeypatch): """ChatClient.ask() sets conversation_id on the returned ChatResult.""" calls = [] + # Set rather than inherited: GOODDATA_EVAL_KEEP_CONVERSATIONS=1 would skip the DELETE. + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) def handler(request): if request.method == "POST" and request.url.path.endswith("/conversations"): @@ -908,6 +912,214 @@ def handler(request): assert "delete" not in calls # conversation preserved +@pytest.fixture +def keep_conversations(monkeypatch: pytest.MonkeyPatch) -> None: + """Turn the process-wide keep flag on for one test, and off again afterwards.""" + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", True) + + +def _delete_recording_handler( + calls: list[str], conversation_id: str, sse_body: bytes +) -> Callable[[httpx.Request], httpx.Response]: + """A transport handler that appends to `calls` on DELETE, so a test can assert on it.""" + + def handler(request: httpx.Request) -> httpx.Response: + if request.method == "POST" and request.url.path.endswith("/conversations"): + return httpx.Response(200, json={"conversationId": conversation_id}) + if request.method == "POST" and "messages" in str(request.url): + return httpx.Response(200, content=sse_body) + if request.method == "DELETE": + calls.append("delete") + return httpx.Response(204) + return httpx.Response(404) + + return handler + + +def test_keep_conversations_keeps_the_conversation_on_success(keep_conversations: None) -> None: + """A passing run is kept too -- that is the whole difference from preserve_failed.""" + calls: list[str] = [] + client = _client_with_handler(_delete_recording_handler(calls, "conv-keep-ok", _OK_SSE)) + + item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) + result = client.ask(item) + + assert result.conversation_id == "conv-keep-ok" + assert "delete" not in calls + + +def test_keep_conversations_keeps_the_conversation_on_failure( + monkeypatch: pytest.MonkeyPatch, keep_conversations: None +) -> None: + """Set on its own, with preserve_failed left off, it still keeps a failed conversation.""" + calls: list[str] = [] + monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) + client = _client_with_handler(_delete_recording_handler(calls, "conv-keep-fail", _NONRETRY_SSE)) + + item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={}) + with pytest.raises(ChatError): + client.ask(item) + + assert "delete" not in calls + + +def test_keep_conversations_blocks_a_direct_delete_conversation_call(keep_conversations: None) -> None: + """The gate is in delete_conversation itself, which is what the agentic evaluators call. + + Each of them creates its own ChatClient deep in the call tree and deletes by hand in a + finally block -- twenty call sites that never pass through ``ask``. If this stops holding, + --keep-conversations silently covers the single-turn kinds only, which is the gap + --preserve-failed already had. + """ + calls: list[str] = [] + client = _client_with_handler(_delete_recording_handler(calls, "conv-direct", _OK_SSE)) + + client.delete_conversation("conv-direct") + + assert calls == [] + + +def test_set_keep_conversations_affects_an_already_built_client(monkeypatch: pytest.MonkeyPatch) -> None: + """The agentic clients may already exist when the CLI applies the flag.""" + calls: list[str] = [] + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) + client = _client_with_handler(_delete_recording_handler(calls, "conv-late", _OK_SSE)) + + sse_mod.set_keep_conversations(True) + client.delete_conversation("conv-late") + assert calls == [] + + sse_mod.set_keep_conversations(False) + client.delete_conversation("conv-late") + assert calls == ["delete"] + + +def test_a_client_built_under_retention_keeps_after_the_run_scope_exits(monkeypatch: pytest.MonkeyPatch) -> None: + """A worker left running past a Ctrl-C finishes after the CLI restored the default. Its + client was built under the run's retention, so its cleanup must still skip the DELETE.""" + calls: list[str] = [] + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) + + with sse_mod.keep_conversations(True): + client = _client_with_handler(_delete_recording_handler(calls, "conv-straggler", _OK_SSE)) + assert sse_mod._KEEP_CONVERSATIONS is False + + client.delete_conversation("conv-straggler") + assert calls == [] + + # A client built after the scope exited is back on the default and deletes. + later = _client_with_handler(_delete_recording_handler(calls, "conv-next", _OK_SSE)) + later.delete_conversation("conv-next") + assert calls == ["delete"] + + +def test_send_message_attaches_conversation_id_to_the_exception(monkeypatch: pytest.MonkeyPatch) -> None: + """The agentic evaluators call send_message directly, not ask(); the runner reads the id + off the exception, so without it a kept conversation is unfindable from the report.""" + calls: list[str] = [] + monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) + client = _client_with_handler(_delete_recording_handler(calls, "conv-send-fail", _NONRETRY_SSE)) + + with pytest.raises(ChatError) as excinfo: + client.send_message("conv-send-fail", "q") + + assert getattr(excinfo.value, "conversation_id", None) == "conv-send-fail" + + +def test_keep_conversations_restores_the_previous_setting(monkeypatch: pytest.MonkeyPatch) -> None: + """One run asking to keep must not leave the next run keeping too.""" + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) + + with sse_mod.keep_conversations(True): + assert sse_mod._retention_on() is True + # The run override never touches the process default. + assert sse_mod._KEEP_CONVERSATIONS is False + assert sse_mod._retention_on() is False + + with pytest.raises(RuntimeError), sse_mod.keep_conversations(True): + raise RuntimeError("run blew up") + assert sse_mod._retention_on() is False + + +def test_keep_conversations_false_does_not_clear_an_env_set_flag(monkeypatch: pytest.MonkeyPatch) -> None: + """``keep=False`` means "did not ask", not "delete" -- an exported env var survives a + run that passes no flag.""" + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", True) + + with sse_mod.keep_conversations(False): + assert sse_mod._retention_on() is True + assert sse_mod._retention_on() is True + assert sse_mod._KEEP_CONVERSATIONS is True + + +def test_overlapping_runs_each_keep_only_their_own_retention(monkeypatch: pytest.MonkeyPatch) -> None: + """A flagged run and a bare run in one process, the bare one starting inside the flagged + one's scope and exiting after it. The bare run must still delete, and once both have + exited nothing may be left on: neither run can see or restore the other's override.""" + calls: list[str] = [] + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) + flagged_entered = threading.Event() + flagged_may_exit = threading.Event() + seen_by_flagged: list[bool] = [] + + def flagged_run() -> None: + with sse_mod.keep_conversations(True): + seen_by_flagged.append(sse_mod._retention_on()) + flagged_entered.set() + flagged_may_exit.wait(timeout=5) + + flagged = threading.Thread(target=flagged_run) + flagged.start() + assert flagged_entered.wait(timeout=5) + + with sse_mod.keep_conversations(False): + assert sse_mod._retention_on() is False + bare_client = _client_with_handler(_delete_recording_handler(calls, "conv-bare", _OK_SSE)) + flagged_may_exit.set() + flagged.join(timeout=5) + bare_client.delete_conversation("conv-bare") + + assert seen_by_flagged == [True] + assert calls == ["delete"] + assert sse_mod._retention_on() is False + + +def test_bind_run_retention_carries_the_override_into_a_pool_thread(monkeypatch: pytest.MonkeyPatch) -> None: + """A pool worker starts without the submitting thread's ContextVars, and a pooled thread + reused afterwards must not keep the override it was given for an earlier task.""" + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) + with ThreadPoolExecutor(max_workers=1) as pool: + with sse_mod.keep_conversations(True): + unbound = pool.submit(sse_mod._retention_on).result() + bound = pool.submit(sse_mod.bind_run_retention(sse_mod._retention_on)).result() + afterwards = pool.submit(sse_mod._retention_on).result() + + assert (unbound, bound, afterwards) == (False, True, False) + + +@pytest.mark.parametrize( + ("raw", "expected"), + [("1", True), ("true", True), ("yes", True), ("0", False), ("false", False), ("off", False), ("", False)], +) +def test_keep_conversations_env_var(monkeypatch: pytest.MonkeyPatch, raw: str, expected: bool) -> None: + """``GOODDATA_EVAL_KEEP_CONVERSATIONS=0`` turns it off rather than reading as truthy.""" + monkeypatch.setenv("GOODDATA_EVAL_KEEP_CONVERSATIONS", raw) + assert sse_mod._bool_env("GOODDATA_EVAL_KEEP_CONVERSATIONS", False) is expected + + +def test_conversations_are_deleted_when_the_flag_is_off(monkeypatch: pytest.MonkeyPatch) -> None: + """The default is unchanged: a run that does not ask to keep state leaves none behind.""" + calls: list[str] = [] + # Set explicitly rather than assumed: with GOODDATA_EVAL_KEEP_CONVERSATIONS exported, the + # module initialises to True and this test would fail when run on its own. + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) + client = _client_with_handler(_delete_recording_handler(calls, "conv-default", _OK_SSE)) + + client.delete_conversation("conv-default") + + assert calls == ["delete"] + + def test_ask_without_preserve_failed_deletes_on_error(monkeypatch): """Without preserve_failed, conversations are deleted even on error.""" calls = [] @@ -923,6 +1135,8 @@ def handler(request): return httpx.Response(404) monkeypatch.setattr(sse_mod.time, "sleep", lambda s: None) + # Set rather than inherited: GOODDATA_EVAL_KEEP_CONVERSATIONS=1 would skip the DELETE. + monkeypatch.setattr(sse_mod, "_KEEP_CONVERSATIONS", False) client = _client_with_handler(handler) item = DatasetItem(id="t1", dataset_name="d", test_kind="visualization", question="q", expected_output={})