From 35e2f629ba96c92205d27146b1adf01810b0e3ac Mon Sep 17 00:00:00 2001 From: EnjoyBacon7 <59032058+EnjoyBacon7@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:30:07 +0000 Subject: [PATCH] feat(evaluation): generate the two promptfoo configs a run executes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run produces two configs, not one, because the two questions need different endpoints: - retrieval hits `GET /search/partition/{partition}`, whose documents carry the chunk `content` that `context-relevance` grades and the `metadata.file_id` that feeds hit rate / MRR / recall; - answers hit `POST /v1/chat/completions`, whose message content is what `factuality` and `llm-rubric` grade. Keeping them separate means every assertion in a config applies to that config's single provider, so no assertion ever runs against an output shape it cannot read. Notes: - `transformResponse` must be a single JavaScript expression. A statement or an IIFE makes promptfoo error every row before grading, so the chat transform extracts the message content and nothing more. - The grader is OpenRAG's own configured LLM endpoint, so model-graded assertions need no third-party credentials. - Questions are interpolated through Nunjucks' `urlencode`, so one containing `&` or `?` cannot corrupt the search query string. - Assertions are copied per test rather than shared: a shared list serialises as a YAML anchor plus aliases. Pure — returns plain dicts. Serialisation and execution live in the worker. --- openrag/core/evaluation/__init__.py | 3 + openrag/core/evaluation/promptfoo_config.py | 171 ++++++++++++++++++ .../core/evaluation/test_promptfoo_config.py | 99 ++++++++++ 3 files changed, 273 insertions(+) create mode 100644 openrag/core/evaluation/promptfoo_config.py create mode 100644 tests/unit/core/evaluation/test_promptfoo_config.py diff --git a/openrag/core/evaluation/__init__.py b/openrag/core/evaluation/__init__.py index 853bc79fd..840afc1f9 100644 --- a/openrag/core/evaluation/__init__.py +++ b/openrag/core/evaluation/__init__.py @@ -2,9 +2,12 @@ from core.evaluation.identity import sanitize_file_id from core.evaluation.metrics import extract_results, indexing_metrics, summarize +from core.evaluation.promptfoo_config import build_answer_config, build_retrieval_config from core.evaluation.testset import parse_testset __all__ = [ + "build_answer_config", + "build_retrieval_config", "extract_results", "indexing_metrics", "parse_testset", diff --git a/openrag/core/evaluation/promptfoo_config.py b/openrag/core/evaluation/promptfoo_config.py new file mode 100644 index 000000000..bee85ed43 --- /dev/null +++ b/openrag/core/evaluation/promptfoo_config.py @@ -0,0 +1,171 @@ +"""Generation of the promptfoo configs a run executes. + +A run produces two configs rather than one, because the two questions need +different endpoints: + +* **retrieval** hits ``GET /search/partition/{partition}``, whose documents + carry the chunk ``content`` — the text that ``context-relevance`` grades and + whose ``metadata.file_id`` feeds hit rate / MRR / recall. +* **answer** hits ``POST /v1/chat/completions``, whose ``extra.sources`` carry + source metadata but no chunk text, and whose message content is what + ``factuality`` and ``llm-rubric`` grade. + +Keeping them separate means every assertion in a config applies to that +config's single provider, so no assertion ever runs against an output shape it +cannot read. + +This module is pure: it returns plain dicts. Serialisation and execution live +in the worker. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from core.models.evaluation import EvalTestCase + +#: promptfoo templates with Nunjucks; ``urlencode`` keeps a question +#: containing ``&`` or ``?`` from corrupting the search query string. +_QUERY_TEMPLATE = "{{ query | urlencode }}" + +#: Extract the ``documents`` array from the search response. +_SEARCH_TRANSFORM = "json.documents || []" + +#: ``transformResponse`` must be a single JavaScript expression — statements +#: and IIFEs are rejected — so this extracts the answer text and nothing more. +#: Retrieved sources come from the retrieval pass instead. +_CHAT_TRANSFORM = "json.choices[0].message.content" + +_RUBRIC = ( + "The response must answer the question using the retrieved documents. " + "Grade it against this reference answer: {{expected_answer}}. " + "Pass if the response conveys the same facts, even if worded differently. " + "Fail if it contradicts the reference, is empty, or refuses to answer." +) + + +def _grader(model: str, base_url: str, api_key: str | None) -> dict[str, Any]: + """The provider promptfoo uses for model-graded assertions. + + Points at OpenRAG's own OpenAI-compatible LLM endpoint so an eval needs no + third-party credentials. + """ + config: dict[str, Any] = {"apiBaseUrl": base_url} + # vLLM ignores the key but the OpenAI client refuses to send without one. + config["apiKey"] = api_key or "sk-no-key-required" + return {"id": f"openai:chat:{model}", "config": config} + + +def _tests(cases: Sequence[EvalTestCase], asserts: list[dict[str, Any]]) -> list[dict[str, Any]]: + """One promptfoo test per case, all sharing the same assertions. + + ``expected_file_ids`` is deliberately absent from ``vars``: no assertion + reads it. The ranking metrics are computed from the retrieved ids in + ``metrics.summarize``, not by promptfoo. + """ + return [ + { + "vars": {"query": case.query, "expected_answer": case.expected_answer}, + # A fresh copy per test: a shared list would serialise as a YAML + # anchor plus aliases. + "assert": [dict(assertion) for assertion in asserts], + } + for case in cases + ] + + +def build_retrieval_config( + *, + cases: Sequence[EvalTestCase], + api_base_url: str, + partition: str, + token: str, + grader_model: str, + grader_base_url: str, + grader_api_key: str | None = None, + top_k: int = 5, + relevance_threshold: float = 0.0, +) -> dict[str, Any]: + """Config that measures what the retriever returns for each question. + + ``relevance_threshold`` defaults to 0 so ``context-relevance`` records a + score without failing the run; the deterministic ranking metrics are + computed from the same responses afterwards. + """ + url = f"{api_base_url.rstrip('/')}/search/partition/{partition}?text={_QUERY_TEMPLATE}&top_k={top_k}" + return { + "description": f"OpenRAG retrieval eval ({partition})", + "prompts": ["{{query}}"], + "providers": [ + { + "id": "https", + "label": "openrag-retrieval", + "config": { + "url": url, + "method": "GET", + "headers": {"Authorization": f"Bearer {token}"}, + "transformResponse": _SEARCH_TRANSFORM, + }, + } + ], + "defaultTest": {"options": {"provider": _grader(grader_model, grader_base_url, grader_api_key)}}, + "tests": _tests( + cases, + [ + { + "type": "context-relevance", + "contextTransform": "output.map(d => d.content).join('\\n\\n')", + "threshold": relevance_threshold, + } + ], + ), + } + + +def build_answer_config( + *, + cases: Sequence[EvalTestCase], + api_base_url: str, + partition: str, + token: str, + grader_model: str, + grader_base_url: str, + grader_api_key: str | None = None, +) -> dict[str, Any]: + """Config that grades the generated answer against the expected one.""" + return { + "description": f"OpenRAG answer eval ({partition})", + "prompts": ["{{query}}"], + "providers": [ + { + "id": "https", + "label": "openrag-chat", + "config": { + "url": f"{api_base_url.rstrip('/')}/v1/chat/completions", + "method": "POST", + "headers": { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + "body": { + "model": f"openrag-{partition}", + "messages": [{"role": "user", "content": "{{query}}"}], + "stream": False, + }, + "transformResponse": _CHAT_TRANSFORM, + }, + } + ], + "defaultTest": {"options": {"provider": _grader(grader_model, grader_base_url, grader_api_key)}}, + "tests": _tests( + cases, + [ + {"type": "factuality", "value": "{{expected_answer}}"}, + {"type": "llm-rubric", "value": _RUBRIC}, + ], + ), + } + + +__all__ = ["build_answer_config", "build_retrieval_config"] diff --git a/tests/unit/core/evaluation/test_promptfoo_config.py b/tests/unit/core/evaluation/test_promptfoo_config.py new file mode 100644 index 000000000..8fabf4480 --- /dev/null +++ b/tests/unit/core/evaluation/test_promptfoo_config.py @@ -0,0 +1,99 @@ +"""Tests for the generated promptfoo configs.""" + +from __future__ import annotations + +from core.evaluation.promptfoo_config import build_answer_config, build_retrieval_config +from core.models.evaluation import EvalTestCase + +CASES = [ + EvalTestCase(query="What is the refund window?", expected_answer="30 days", expected_file_ids=("p.pdf",)), + EvalTestCase(query="Who approves?", expected_answer="The CFO"), +] + +COMMON = { + "api_base_url": "http://openrag:8080/", + "partition": "__eval_abc", + "token": "or-secret", + "grader_model": "qwen", + "grader_base_url": "http://vllm:8000/v1", +} + + +def test_retrieval_provider_targets_the_single_partition_search_route(): + config = build_retrieval_config(cases=CASES, **COMMON, top_k=7) + url = config["providers"][0]["config"]["url"] + assert url.startswith("http://openrag:8080/search/partition/__eval_abc") + assert "top_k=7" in url + + +def test_retrieval_query_is_url_encoded(): + """A question containing '&' would otherwise truncate the query string.""" + config = build_retrieval_config(cases=CASES, **COMMON) + assert "{{ query | urlencode }}" in config["providers"][0]["config"]["url"] + + +def test_retrieval_asserts_on_the_chunk_text(): + config = build_retrieval_config(cases=CASES, **COMMON) + assertion = config["tests"][0]["assert"][0] + assert assertion["type"] == "context-relevance" + assert "d.content" in assertion["contextTransform"] + + +def test_answer_provider_posts_to_the_partition_scoped_model(): + config = build_answer_config(cases=CASES, **COMMON) + body = config["providers"][0]["config"]["body"] + assert config["providers"][0]["config"]["url"] == "http://openrag:8080/v1/chat/completions" + assert body["model"] == "openrag-__eval_abc" + assert body["stream"] is False + + +def test_answer_transform_is_a_single_expression(): + """promptfoo evaluates transformResponse as an expression — a statement or + an IIFE fails at runtime with a transform error, which manifests as every + answer scoring zero.""" + transform = build_answer_config(cases=CASES, **COMMON)["providers"][0]["config"]["transformResponse"] + assert transform == "json.choices[0].message.content" + assert "return" not in transform + assert ";" not in transform + + +def test_answer_grades_against_the_expected_answer(): + config = build_answer_config(cases=CASES, **COMMON) + types = [assertion["type"] for assertion in config["tests"][0]["assert"]] + assert types == ["factuality", "llm-rubric"] + assert config["tests"][0]["assert"][0]["value"] == "{{expected_answer}}" + + +def test_both_configs_send_the_bearer_token(): + for config in ( + build_retrieval_config(cases=CASES, **COMMON), + build_answer_config(cases=CASES, **COMMON), + ): + headers = config["providers"][0]["config"]["headers"] + assert headers["Authorization"] == "Bearer or-secret" + + +def test_grader_points_at_the_configured_openrag_llm(): + """Model-graded assertions must not silently fall back to OpenAI.""" + config = build_answer_config(cases=CASES, **COMMON) + grader = config["defaultTest"]["options"]["provider"] + assert grader["id"] == "openai:chat:qwen" + assert grader["config"]["apiBaseUrl"] == "http://vllm:8000/v1" + assert grader["config"]["apiKey"] + + +def test_every_case_becomes_a_test_with_its_vars(): + """Only the vars an assertion actually templates are emitted — the ranking + metrics read expected_file_ids from the test set, not from promptfoo.""" + config = build_retrieval_config(cases=CASES, **COMMON) + assert len(config["tests"]) == 2 + assert config["tests"][0]["vars"] == { + "query": "What is the refund window?", + "expected_answer": "30 days", + } + + +def test_assertions_are_not_shared_between_tests(): + """A shared list would serialise as a YAML anchor plus aliases.""" + tests = build_answer_config(cases=CASES, **COMMON)["tests"] + assert tests[0]["assert"][0] is not tests[1]["assert"][0]