diff --git a/README.md b/README.md index 92a460f..93e213d 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,12 @@ A comprehensive framework for evaluating Retrieval-Augmented Generation (RAG) mo - **Context Precision**: Measures how relevant the retrieved context is - **Context Recall**: Measures if all relevant information is retrieved +- **Qualitative Answer Logging**: + - Side-by-side comparison of RAG-augmented answers vs direct LLM answers + - Logs question category, model name, retrieved context, and both answers + - Outputs to CSV (for Excel/Sheets review) and JSON (for programmatic analysis) + - Optional evaluation scores attached to each log entry + - **Data Ingestion Support**: - Jabref (BibTeX) format - DataTable formats (CSV, JSON, Excel) @@ -24,6 +30,7 @@ A comprehensive framework for evaluating Retrieval-Augmented Generation (RAG) mo - Single evaluation - Batch evaluation - Custom metric selection + - Qualitative logging with structured output ## Installation @@ -126,6 +133,90 @@ data = jabref_loader.load_for_evaluation('references.bib') results = evaluator.evaluate_batch(**data) ``` +### Qualitative Answer Logging + +Log RAG answers alongside direct LLM answers for side-by-side qualitative analysis. This helps you compare what the model answers **with** retrieved context versus **without** it. + +#### From Python + +```python +from rag_evaluation import QualitativeLogger, LogEntry + +logger = QualitativeLogger() + +logger.log(LogEntry( + category="factual", + model_name="gemini-1.5-flash", + question="What is data.table?", + rag_context="data.table is an R package that provides an enhanced version of data.frame...", + rag_answer="data.table is an R package that extends data.frame with fast aggregation...", + llm_answer="data.table is a popular R package used for data manipulation...", +)) + +# Save to both CSV and JSON +written = logger.save(output_dir="logs") +print(written) +# {'csv': 'logs/qualitative_log_2026-02-11_143022.csv', +# 'json': 'logs/qualitative_log_2026-02-11_143022.json'} +``` + +#### Attaching Evaluation Scores + +You can compute evaluation metrics and attach them to each log entry: + +```python +from rag_evaluation import RAGEvaluator, QualitativeLogger, LogEntry + +evaluator = RAGEvaluator() +scores = evaluator.evaluate( + query="What is data.table?", + context="data.table is an R package...", + answer="data.table is an R package that extends...", +) + +logger = QualitativeLogger() +logger.log(LogEntry( + category="factual", + model_name="gemini-1.5-flash", + question="What is data.table?", + rag_context="data.table is an R package...", + rag_answer="data.table is an R package that extends...", + llm_answer="data.table is a popular R package...", + evaluation_scores=scores, +)) +logger.save("logs") +``` + +#### From the Command Line + +The `examples/qualitative_eval.py` script provides a full CLI: + +```bash +# Basic: load data and log to CSV + JSON +python examples/qualitative_eval.py your_data.csv + +# With evaluation scores and verbose per-entry output +python examples/qualitative_eval.py your_data.csv --output-dir results/logs --with-scores --verbose + +# Override the model name for all entries +python examples/qualitative_eval.py your_data.csv --model-name gemini-2.0-flash + +# Choose specific metrics when scoring +python examples/qualitative_eval.py your_data.csv --with-scores --metrics faithfulness relevance +``` + +**CLI arguments:** + +| Argument | Description | +|---|---| +| `data_file` | Path to input data file (CSV, JSON, or Excel) | +| `--type` | Input format: `csv`, `json`, `excel`, or `auto` (default: `auto`) | +| `--output-dir` | Directory for log files (default: `logs/`) | +| `--model-name` | Override the model name for all entries | +| `--with-scores` | Compute evaluation metrics and attach to each entry | +| `--metrics` | Which metrics to compute: `faithfulness`, `context_precision`, `relevance` | +| `--verbose`, `-v` | Print detailed per-entry output to the console | + ## Evaluation Metrics ### Basic Evaluator Metrics (Rule-based) @@ -179,14 +270,14 @@ The ragas evaluator uses advanced LLM-based evaluation for more nuanced assessme ## Data Format -### CSV Format +### Evaluation Data (CSV) ```csv query,context,answer,ground_truth "What is ML?","ML is...","ML allows...","ML is..." ``` -### JSON Format +### Evaluation Data (JSON) ```json [ @@ -199,6 +290,42 @@ query,context,answer,ground_truth ] ``` +### Qualitative Logging Data (CSV) + +For the qualitative logger, provide a file with these columns: + +```csv +category,model_name,question,rag_context,rag_answer,llm_answer +factual,gemini-1.5-flash,"What is data.table?","data.table is...","data.table is a package...","data.table is a library..." +code,gemini-1.5-flash,"How to read CSV?","Use fread()...","Use fread() to read...","Use read.csv() to read..." +``` + +| Column | Description | +|---|---| +| `category` | Question category for grouping (e.g. `factual`, `reasoning`, `code`) | +| `model_name` | LLM model used (e.g. `gemini-1.5-flash`) | +| `question` | The original user question | +| `rag_context` | The retrieved context that was passed to the LLM | +| `rag_answer` | The answer generated by the LLM **with** RAG context | +| `llm_answer` | The answer from the LLM **without** RAG (direct response) | + +A sample file is provided at `examples/sample_qualitative_data.csv`. + +### Qualitative Logging Data (JSON) + +```json +[ + { + "category": "factual", + "model_name": "gemini-1.5-flash", + "question": "What is data.table?", + "rag_context": "data.table is...", + "rag_answer": "data.table is a package...", + "llm_answer": "data.table is a library..." + } +] +``` + ### BibTeX Format (Jabref) ```bibtex @@ -216,9 +343,11 @@ Check the `examples/` directory for complete usage examples: - `examples/basic_usage.py`: Comprehensive examples of basic evaluator - `examples/ragas_usage.py`: Examples using ragas evaluator with LLM-based metrics - `examples/evaluate.py`: Command-line evaluation tool -- `examples/sample_data.json`: Sample JSON data -- `examples/sample_data.csv`: Sample CSV data +- `examples/qualitative_eval.py`: Command-line qualitative logging tool (RAG vs LLM comparison) +- `examples/sample_data.json`: Sample JSON evaluation data +- `examples/sample_data.csv`: Sample CSV evaluation data - `examples/sample_data.bib`: Sample BibTeX data +- `examples/sample_qualitative_data.csv`: Sample CSV for qualitative logging Run the examples: @@ -226,6 +355,12 @@ Run the examples: cd examples python basic_usage.py +# Qualitative logging (no API key required for basic logging) +python qualitative_eval.py sample_qualitative_data.csv --verbose + +# Qualitative logging with evaluation scores +python qualitative_eval.py sample_qualitative_data.csv --with-scores --verbose + # For ragas examples (requires OpenAI API key) export OPENAI_API_KEY='your-key-here' python ragas_usage.py @@ -235,17 +370,18 @@ python ragas_usage.py ``` rag_evaluation/ -├── __init__.py # Main package exports -├── evaluator.py # RAGEvaluator class (basic, rule-based) -├── ragas_evaluator.py # RagasEvaluator class (LLM-based) +├── __init__.py # Main package exports +├── evaluator.py # RAGEvaluator class (basic, rule-based) +├── ragas_evaluator.py # RagasEvaluator class (LLM-based) +├── qualitative_logger.py # QualitativeLogger + LogEntry for answer logging ├── metrics/ │ ├── __init__.py -│ ├── faithfulness.py # Faithfulness metric -│ ├── context_precision.py # Context precision metric -│ └── relevance.py # Relevance metric +│ ├── faithfulness.py # Faithfulness metric +│ ├── context_precision.py # Context precision metric +│ └── relevance.py # Relevance metric └── data_ingestion/ ├── __init__.py - ├── jabref_loader.py # Jabref/BibTeX loader + ├── jabref_loader.py # Jabref/BibTeX loader └── datatable_loader.py # CSV/JSON/Excel loader ``` @@ -285,6 +421,33 @@ Evaluator using the ragas library for LLM-based evaluation metrics. - `evaluate_batch(queries, contexts, answers, ground_truths=None)`: Evaluate multiple outputs - `get_average_scores(batch_results)`: Calculate average scores from batch results +### QualitativeLogger + +Accumulates log entries and writes them to CSV and/or JSON for qualitative analysis. + +**Methods**: +- `log(entry)`: Append a single `LogEntry` +- `log_batch(entries)`: Append multiple `LogEntry` objects at once +- `save(output_dir="logs", formats=["csv", "json"])`: Write logs to disk; returns dict of `{format: filepath}` + +**Properties**: +- `entries`: List of accumulated `LogEntry` objects +- `len(logger)`: Number of entries logged so far + +### LogEntry + +Pydantic model representing a single qualitative log record. + +**Fields**: +- `timestamp` (str): Auto-generated ISO timestamp +- `category` (str): Question category (e.g. `"factual"`, `"reasoning"`, `"code"`) +- `model_name` (str): LLM model name (e.g. `"gemini-1.5-flash"`) +- `question` (str): The original user question +- `rag_context` (str): Retrieved context passed to the LLM +- `rag_answer` (str): LLM answer generated with RAG context +- `llm_answer` (str): LLM answer generated without RAG +- `evaluation_scores` (dict, optional): Metric scores from `RAGEvaluator` + ### DataTableLoader Loader for tabular data formats. @@ -292,6 +455,7 @@ Loader for tabular data formats. **Methods**: - `load(file_path, format=None)`: Load data from file - `load_for_evaluation(file_path, ...)`: Load data ready for evaluation +- `load_for_qualitative_logging(file_path, ...)`: Load data ready for qualitative logging (columns: `category`, `model_name`, `question`, `rag_context`, `rag_answer`, `llm_answer`) **Supported Formats**: CSV, JSON, Excel (.xlsx, .xls) diff --git a/examples/qualitative_eval.py b/examples/qualitative_eval.py new file mode 100644 index 0000000..ad04bdb --- /dev/null +++ b/examples/qualitative_eval.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +Qualitative RAG Evaluation Runner + +Loads a data file containing questions, RAG answers, and direct LLM answers, +then logs them in both CSV and JSON for side-by-side qualitative analysis. + +Optionally computes evaluation metric scores and attaches them to each log entry. + +Usage: + python qualitative_eval.py data.csv + python qualitative_eval.py data.csv --output-dir results/logs --with-scores --verbose + python qualitative_eval.py data.json --model-name gemini-1.5-flash +""" + +import argparse +import sys +from pathlib import Path + +# Add parent directory to path to import rag_evaluation +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from rag_evaluation import RAGEvaluator, QualitativeLogger, LogEntry +from rag_evaluation.data_ingestion import DataTableLoader + + +def load_qualitative_data(file_path: str, data_type: str = "auto") -> dict: + """ + Load data from file using DataTableLoader. + + Args: + file_path: Path to the data file + data_type: Format hint ('csv', 'json', 'excel', or 'auto') + + Returns: + Dictionary with lists of categories, model_names, questions, + rag_contexts, rag_answers, and llm_answers + """ + path = Path(file_path) + if not path.exists(): + raise FileNotFoundError(f"Data file not found: {file_path}") + + fmt = None if data_type == "auto" else data_type + loader = DataTableLoader() + return loader.load_for_qualitative_logging(file_path, format=fmt) + + +def build_log_entries( + data: dict, + model_name_override: str | None = None, + evaluator: RAGEvaluator | None = None, +) -> list[LogEntry]: + """ + Convert loaded data rows into LogEntry objects. + + If an evaluator is provided, each entry will include evaluation scores + computed from (question, rag_context, rag_answer). + + Args: + data: Output of load_qualitative_data() + model_name_override: If set, overrides the model_name for every entry + evaluator: Optional RAGEvaluator for attaching metric scores + + Returns: + List of LogEntry objects ready for the logger + """ + entries: list[LogEntry] = [] + n = len(data["questions"]) + + for i in range(n): + model = model_name_override or data["model_names"][i] + question = data["questions"][i] + rag_context = data["rag_contexts"][i] + rag_answer = data["rag_answers"][i] + + scores = None + if evaluator and rag_context and rag_answer: + scores = evaluator.evaluate( + query=question, + context=rag_context, + answer=rag_answer, + ) + + entries.append( + LogEntry( + category=data["categories"][i], + model_name=model, + question=question, + rag_context=rag_context, + rag_answer=rag_answer, + llm_answer=data["llm_answers"][i], + evaluation_scores=scores, + ) + ) + + return entries + + +def print_summary(entries: list[LogEntry], verbose: bool = False) -> None: + """Print a human-readable summary of the logged entries.""" + print("\n" + "=" * 70) + print("QUALITATIVE LOG SUMMARY") + print("=" * 70) + print(f" Total entries: {len(entries)}") + + # Category breakdown + categories = {} + for e in entries: + cat = e.category or "(uncategorized)" + categories[cat] = categories.get(cat, 0) + 1 + if categories: + print(" Categories:") + for cat, count in sorted(categories.items()): + print(f" - {cat}: {count}") + + # Model breakdown + models = {} + for e in entries: + m = e.model_name or "(unknown)" + models[m] = models.get(m, 0) + 1 + if models: + print(" Models:") + for m, count in sorted(models.items()): + print(f" - {m}: {count}") + + if verbose: + print("\n" + "-" * 70) + for i, entry in enumerate(entries, 1): + print(f"\n [{i}] {entry.category or '-'} | {entry.model_name or '-'}") + print(f" Q: {entry.question[:80]}{'...' if len(entry.question) > 80 else ''}") + print(f" RAG: {entry.rag_answer[:80]}{'...' if len(entry.rag_answer) > 80 else ''}") + print(f" LLM: {entry.llm_answer[:80]}{'...' if len(entry.llm_answer) > 80 else ''}") + if entry.evaluation_scores: + scores_str = ", ".join( + f"{k}: {v.get('score', v) if isinstance(v, dict) else v:.3f}" + for k, v in entry.evaluation_scores.items() + if (isinstance(v, dict) and v.get("score") is not None) or isinstance(v, (int, float)) + ) + if scores_str: + print(f" Scores: {scores_str}") + + print("\n" + "=" * 70) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Log RAG vs direct LLM answers for qualitative analysis." + ) + + parser.add_argument( + "data_file", + help="Path to the input data file (CSV, JSON, or Excel)", + ) + parser.add_argument( + "--type", + choices=["csv", "json", "excel", "auto"], + default="auto", + help="Input file format (default: auto-detect from extension)", + ) + parser.add_argument( + "--output-dir", + default="logs", + help="Directory to write log files into (default: logs/)", + ) + parser.add_argument( + "--model-name", + default=None, + help="Override model name for all entries (uses data column value if omitted)", + ) + parser.add_argument( + "--with-scores", + action="store_true", + help="Compute evaluation metric scores and attach to each log entry", + ) + parser.add_argument( + "--metrics", + nargs="+", + choices=["faithfulness", "context_precision", "relevance"], + default=None, + help="Metrics to compute when --with-scores is used (default: all)", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print detailed per-entry output", + ) + + args = parser.parse_args() + + try: + # Load data + print(f"Loading data from: {args.data_file}") + data = load_qualitative_data(args.data_file, args.type) + n = len(data["questions"]) + print(f"Loaded {n} entries") + + # Optionally set up evaluator + evaluator = None + if args.with_scores: + evaluator = RAGEvaluator(metrics=args.metrics) + print(f"Scoring with metrics: {list(evaluator.metrics.keys())}") + + # Build log entries + entries = build_log_entries(data, args.model_name, evaluator) + + # Log and save + logger = QualitativeLogger() + logger.log_batch(entries) + + written = logger.save(output_dir=args.output_dir) + for fmt, path in written.items(): + print(f" {fmt.upper()} saved to: {path}") + + # Print summary + print_summary(entries, verbose=args.verbose) + + return 0 + + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/sample_qualitative_data.csv b/examples/sample_qualitative_data.csv new file mode 100644 index 0000000..2106a08 --- /dev/null +++ b/examples/sample_qualitative_data.csv @@ -0,0 +1,4 @@ +category,model_name,question,rag_context,rag_answer,llm_answer +factual,gemini-1.5-flash,What is data.table in R?,"data.table is an R package that provides an enhanced version of data.frame. It offers fast aggregation of large datasets, ordered joins, and the ability to update columns by reference.","data.table is an R package that extends data.frame with fast aggregation, ordered joins, and update-by-reference capabilities for large datasets.","data.table is a popular R package used for data manipulation. It is known for being fast and memory-efficient, often used as an alternative to dplyr for working with large datasets." +code,gemini-1.5-flash,How do you create a data.table from a CSV file?,"Use fread() to read CSV files: library(data.table); dt <- fread('file.csv'). fread() automatically detects separators, column types, and is significantly faster than read.csv().","You can use the fread() function: library(data.table); dt <- fread('file.csv'). It auto-detects separators and column types and is much faster than base R's read.csv().","To read a CSV file in R, you can use read.csv('file.csv') or the readr package's read_csv('file.csv'). For large files, the data.table package offers fread() which is faster." +reasoning,gemini-1.5-flash,Why is data.table faster than dplyr for large datasets?,"data.table uses several optimizations: radix-based ordering, binary search for joins, column references to avoid copies, and automatic indexing. It modifies data in-place using the := operator which avoids memory allocation overhead.","data.table is faster because it modifies data in-place with :=, uses radix-based ordering, binary search joins, and automatic indexing. These avoid the memory allocation overhead that comes with creating copies.","data.table is generally faster than dplyr for large datasets because it is written in C and optimizes memory usage. It uses efficient algorithms for grouping, joining, and sorting operations." diff --git a/pyproject.toml b/pyproject.toml index 8a07a76..391ce8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.10" license = { text = "MIT" } dependencies = [ + "pydantic>=2.0.0", "ragas>=0.1.0", ] diff --git a/rag_evaluation/__init__.py b/rag_evaluation/__init__.py index e4fb18f..3e67fd5 100644 --- a/rag_evaluation/__init__.py +++ b/rag_evaluation/__init__.py @@ -9,6 +9,7 @@ from .metrics.faithfulness import FaithfulnessMetric from .metrics.context_precision import ContextPrecisionMetric from .metrics.relevance import RelevanceMetric +from .qualitative_logger import QualitativeLogger, LogEntry # Try to import RagasEvaluator (optional dependency) try: @@ -20,6 +21,8 @@ "ContextPrecisionMetric", "RelevanceMetric", "RagasEvaluator", + "QualitativeLogger", + "LogEntry", ] except ImportError: _RAGAS_AVAILABLE = False @@ -29,6 +32,8 @@ "FaithfulnessMetric", "ContextPrecisionMetric", "RelevanceMetric", + "QualitativeLogger", + "LogEntry", ] __version__ = "0.1.0" diff --git a/rag_evaluation/data_ingestion/datatable_loader.py b/rag_evaluation/data_ingestion/datatable_loader.py index 5f38f8b..8d2fd71 100644 --- a/rag_evaluation/data_ingestion/datatable_loader.py +++ b/rag_evaluation/data_ingestion/datatable_loader.py @@ -173,3 +173,46 @@ def load_for_evaluation( 'answers': [e.get(answer_column, '') for e in entries], 'ground_truths': [e.get(ground_truth_column, '') for e in entries] } + + def load_for_qualitative_logging( + self, + file_path: str, + category_column: str = 'category', + model_name_column: str = 'model_name', + query_column: str = 'question', + rag_context_column: str = 'rag_context', + rag_answer_column: str = 'rag_answer', + llm_answer_column: str = 'llm_answer', + format: Optional[str] = None + ) -> Dict[str, List[str]]: + """ + Load data in a format ready for qualitative logging. + + Expects a tabular file with columns for the question category, + model name, the question itself, the RAG-retrieved context, + the RAG-augmented answer, and the direct LLM answer. + + Args: + file_path: Path to the data file + category_column: Name of the category column + model_name_column: Name of the model name column + query_column: Name of the question/query column + rag_context_column: Name of the RAG context column + rag_answer_column: Name of the RAG answer column + llm_answer_column: Name of the direct LLM answer column + format: Optional format specifier + + Returns: + Dictionary with lists of categories, model_names, questions, + rag_contexts, rag_answers, and llm_answers + """ + entries = self.load(file_path, format) + + return { + 'categories': [e.get(category_column, '') for e in entries], + 'model_names': [e.get(model_name_column, '') for e in entries], + 'questions': [e.get(query_column, '') for e in entries], + 'rag_contexts': [e.get(rag_context_column, '') for e in entries], + 'rag_answers': [e.get(rag_answer_column, '') for e in entries], + 'llm_answers': [e.get(llm_answer_column, '') for e in entries], + } diff --git a/rag_evaluation/qualitative_logger.py b/rag_evaluation/qualitative_logger.py new file mode 100644 index 0000000..ab7ff31 --- /dev/null +++ b/rag_evaluation/qualitative_logger.py @@ -0,0 +1,193 @@ +""" +Qualitative Logger for RAG Evaluation + +Provides structured logging of RAG vs direct LLM answers for qualitative analysis. +Outputs to both CSV (for human review in Excel/Sheets) and JSON (for programmatic analysis). +""" + +import csv +import json +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Any + +from pydantic import BaseModel, Field + + +class LogEntry(BaseModel): + """ + Schema for a single qualitative log entry. + + Captures the full picture of a RAG query: the question, what was retrieved, + what the RAG-augmented LLM answered, and what the LLM answered on its own. + """ + + timestamp: str = Field( + default_factory=lambda: datetime.now().isoformat(timespec="seconds"), + description="ISO-formatted timestamp of when the entry was logged", + ) + category: str = Field( + default="", + description="Question category (e.g. 'factual', 'reasoning', 'code')", + ) + model_name: str = Field( + default="", + description="Name of the LLM model used (e.g. 'gemini-1.5-flash')", + ) + question: str = Field( + description="The original user question / query", + ) + rag_context: str = Field( + default="", + description="The retrieved context that was passed to the LLM", + ) + rag_answer: str = Field( + default="", + description="The answer generated by the LLM using RAG context", + ) + llm_answer: str = Field( + default="", + description="The answer from the LLM without RAG (direct response)", + ) + evaluation_scores: Optional[Dict[str, Any]] = Field( + default=None, + description="Optional evaluation metric scores from RAGEvaluator", + ) + + +# Column order for CSV output (evaluation_scores is flattened into individual columns) +_CSV_BASE_COLUMNS = [ + "timestamp", + "category", + "model_name", + "question", + "rag_context", + "rag_answer", + "llm_answer", +] + + +class QualitativeLogger: + """ + Accumulates qualitative log entries and saves them as CSV and/or JSON. + + Usage: + logger = QualitativeLogger() + logger.log(LogEntry(question="What is X?", rag_answer="...", llm_answer="...")) + logger.save("output/logs") + """ + + def __init__(self) -> None: + self._entries: List[LogEntry] = [] + + @property + def entries(self) -> List[LogEntry]: + """Return a copy of the accumulated entries.""" + return list(self._entries) + + def __len__(self) -> int: + return len(self._entries) + + def log(self, entry: LogEntry) -> None: + """Append a single log entry.""" + self._entries.append(entry) + + def log_batch(self, entries: List[LogEntry]) -> None: + """Append multiple log entries at once.""" + self._entries.extend(entries) + + # ------------------------------------------------------------------ + # Serialisation helpers + # ------------------------------------------------------------------ + + def _collect_score_columns(self) -> List[str]: + """Determine the union of all metric names across entries.""" + columns: dict[str, None] = {} # ordered set + for entry in self._entries: + if entry.evaluation_scores: + for key in entry.evaluation_scores: + columns[key] = None + return list(columns) + + def _entry_to_flat_dict(self, entry: LogEntry, score_columns: List[str]) -> Dict[str, Any]: + """Convert a LogEntry to a flat dict suitable for CSV row writing.""" + row: Dict[str, Any] = {} + for col in _CSV_BASE_COLUMNS: + row[col] = getattr(entry, col) + + scores = entry.evaluation_scores or {} + for col in score_columns: + value = scores.get(col) + # Extract numeric score from metric result dicts + if isinstance(value, dict) and "score" in value: + value = value["score"] + row[col] = value if value is not None else "" + + return row + + def _entry_to_json_dict(self, entry: LogEntry) -> Dict[str, Any]: + """Convert a LogEntry to a dict suitable for JSON serialisation.""" + data = entry.model_dump() + # Strip None evaluation_scores for cleaner output + if data.get("evaluation_scores") is None: + del data["evaluation_scores"] + return data + + # ------------------------------------------------------------------ + # Save + # ------------------------------------------------------------------ + + def save( + self, + output_dir: str = "logs", + formats: Optional[List[str]] = None, + filename_prefix: str = "qualitative_log", + ) -> Dict[str, str]: + """ + Write accumulated entries to disk. + + Args: + output_dir: Directory to write log files into (created if missing). + formats: List of formats to write. Defaults to ``["csv", "json"]``. + filename_prefix: Prefix for the generated filenames. + + Returns: + Dictionary mapping format name to the written file path. + """ + if formats is None: + formats = ["csv", "json"] + + out_path = Path(output_dir) + out_path.mkdir(parents=True, exist_ok=True) + + timestamp_slug = datetime.now().strftime("%Y-%m-%d_%H%M%S") + written: Dict[str, str] = {} + + if "csv" in formats: + csv_file = out_path / f"{filename_prefix}_{timestamp_slug}.csv" + self._write_csv(csv_file) + written["csv"] = str(csv_file) + + if "json" in formats: + json_file = out_path / f"{filename_prefix}_{timestamp_slug}.json" + self._write_json(json_file) + written["json"] = str(json_file) + + return written + + def _write_csv(self, file_path: Path) -> None: + """Write all entries to a CSV file.""" + score_columns = self._collect_score_columns() + fieldnames = _CSV_BASE_COLUMNS + score_columns + + with open(file_path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for entry in self._entries: + writer.writerow(self._entry_to_flat_dict(entry, score_columns)) + + def _write_json(self, file_path: Path) -> None: + """Write all entries to a JSON file.""" + data = [self._entry_to_json_dict(entry) for entry in self._entries] + with open(file_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False)