From 013cfec4c56903bedc2885286c09e1c98d6d2a29 Mon Sep 17 00:00:00 2001 From: Andrew Sliva Date: Wed, 11 Feb 2026 10:48:50 -0700 Subject: [PATCH 1/2] Add logging to data ingestion in docgpt d: systems/docgpt/main.py --- systems/docgpt/main.py | 50 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/systems/docgpt/main.py b/systems/docgpt/main.py index 7be7955..1c94cff 100644 --- a/systems/docgpt/main.py +++ b/systems/docgpt/main.py @@ -1,5 +1,6 @@ from pathlib import Path +import logging import pypandoc from dependency_injector.wiring import Provide, inject from dotenv import load_dotenv @@ -13,6 +14,8 @@ from src.port.assistant import AssistantPort from src.port.content import ContentPort +logger = logging.getLogger(__name__) + @inject def run_terminal( @@ -42,16 +45,57 @@ def add_documents( storage: VectorStore = Provide[containers.Settings.storage.vector_storage], ) -> None: fails_count = 0 + failed_files = [] for doc in documents: try: storage.add_documents([doc]) - except (Exception,) as e: + except Exception as e: fails_count += 1 - print(f"Fail to add document: {e}") + + # Extract file information from metadata + metadata = doc.metadata if hasattr(doc, 'metadata') else {} + file_name = metadata.get('file_name', 'Unknown') + file_path = metadata.get('file_path', metadata.get('source', 'Unknown')) + project = metadata.get('project', 'Unknown') + source = metadata.get('source', 'Unknown') + + # Determine file type from file extension + file_type = 'Unknown' + if file_name and file_name != 'Unknown': + file_type = Path(file_name).suffix or 'No extension' + elif file_path and file_path != 'Unknown': + file_type = Path(file_path).suffix or 'No extension' + + # Get exception details + exception_type = type(e).__name__ + exception_message = str(e) + + # Log detailed error information + logger.error( + f"Failed to ingest file - " + f"File Name: {file_name}, " + f"File Type: {file_type}, " + f"File Path: {file_path}, " + f"Project: {project}, " + f"Source: {source}, " + f"Exception Type: {exception_type}, " + f"Reason: {exception_message}" + ) + + failed_files.append({ + 'file_name': file_name, + 'file_type': file_type, + 'file_path': file_path, + 'project': project, + 'source': source, + 'exception_type': exception_type, + 'reason': exception_message + }) if fails_count: - print(f"{fails_count} documents failed to add") + logger.warning(f"Total of {fails_count} documents failed to ingest") + logger.info(f"Failed files summary: {failed_files}") @inject From db6bf7c39c69c969d5516da6982e7d192db3a0db Mon Sep 17 00:00:00 2001 From: Andrew Sliva Date: Wed, 11 Feb 2026 10:58:21 -0700 Subject: [PATCH 2/2] add github actions --- .github/labeler.yml | 76 +++++++++++++ .github/workflows/auto-label.yml | 87 +++++++++++++++ .github/workflows/ci.yml | 108 +++++++++++++++++++ .github/workflows/codeql.yml | 41 +++++++ .github/workflows/dependency-review.yml | 24 +++++ .github/workflows/pr-changelog.yml | 136 ++++++++++++++++++++++++ .github/workflows/stale.yml | 45 ++++++++ pyproject.toml | 69 ++++++++++++ tests/__init__.py | 0 tests/conftest.py | 65 +++++++++++ tests/test_data_ingestion.py | 94 ++++++++++++++++ tests/test_evaluator.py | 126 ++++++++++++++++++++++ tests/test_metrics.py | 86 +++++++++++++++ 13 files changed, 957 insertions(+) create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/auto-label.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/dependency-review.yml create mode 100644 .github/workflows/pr-changelog.yml create mode 100644 .github/workflows/stale.yml create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_data_ingestion.py create mode 100644 tests/test_evaluator.py create mode 100644 tests/test_metrics.py diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000..4c040f1 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,76 @@ +# Label configuration for actions/labeler +# Maps labels to file path glob patterns + +# Core evaluation framework +evaluation: + - changed-files: + - any-glob-to-any-file: + - "rag_evaluation/**" + +metrics: + - changed-files: + - any-glob-to-any-file: + - "rag_evaluation/metrics/**" + +data-ingestion: + - changed-files: + - any-glob-to-any-file: + - "rag_evaluation/data_ingestion/**" + +# DocGPT subproject +docgpt: + - changed-files: + - any-glob-to-any-file: + - "systems/docgpt/**" + +# Tests +tests: + - changed-files: + - any-glob-to-any-file: + - "tests/**" + - "systems/docgpt/tests/**" + +# Documentation +documentation: + - changed-files: + - any-glob-to-any-file: + - "*.md" + - "docs/**" + - "examples/**" + +# CI/CD +ci: + - changed-files: + - any-glob-to-any-file: + - ".github/**" + +# Dependencies +dependencies: + - changed-files: + - any-glob-to-any-file: + - "requirements.txt" + - "pyproject.toml" + - "setup.py" + - "setup.cfg" + - "systems/docgpt/pyproject.toml" + - "systems/docgpt/uv.lock" + +# Configuration +config: + - changed-files: + - any-glob-to-any-file: + - "*.toml" + - "*.cfg" + - "*.ini" + - "*.yml" + - "*.yaml" + - ".flake8" + - ".pre-commit-config.yaml" + +# Docker +docker: + - changed-files: + - any-glob-to-any-file: + - "**/Dockerfile" + - "**/docker-compose*.yml" + - "**/.docker/**" diff --git a/.github/workflows/auto-label.yml b/.github/workflows/auto-label.yml new file mode 100644 index 0000000..b855daf --- /dev/null +++ b/.github/workflows/auto-label.yml @@ -0,0 +1,87 @@ +name: Auto Label PRs + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + pull-requests: write + contents: read + +jobs: + label-by-files: + name: Label by Changed Files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Label PR by files changed + uses: actions/labeler@v5 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + configuration-path: .github/labeler.yml + + label-by-size: + name: Label by PR Size + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Label PR by size + uses: actions/github-script@v7 + with: + script: | + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + per_page: 100, + }); + + let totalChanges = 0; + for (const file of files) { + totalChanges += file.additions + file.deletions; + } + + let sizeLabel = ''; + if (totalChanges < 10) { + sizeLabel = 'size/XS'; + } else if (totalChanges < 50) { + sizeLabel = 'size/S'; + } else if (totalChanges < 200) { + sizeLabel = 'size/M'; + } else if (totalChanges < 500) { + sizeLabel = 'size/L'; + } else { + sizeLabel = 'size/XL'; + } + + // Remove existing size labels + const { data: currentLabels } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + for (const label of currentLabels) { + if (label.name.startsWith('size/')) { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name: label.name, + }); + } + } + + // Add new size label + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [sizeLabel], + }); + + console.log(`PR has ${totalChanges} changes → labeled as ${sizeLabel}`); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1aaa325 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,108 @@ +name: CI + +on: + push: + branches: [main, master, develop] + pull_request: + branches: [main, master, develop] + +permissions: + contents: read + +jobs: + lint: + name: Lint & Format Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff mypy + + - name: Run Ruff linter + run: ruff check . --output-format=github + + - name: Run Ruff formatter check + run: ruff format --check . + + type-check: + name: Type Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install mypy + pip install -r requirements.txt || true + + - name: Run mypy + run: mypy rag_evaluation/ --ignore-missing-imports + + test: + name: Test (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-cov + pip install -r requirements.txt || true + + - name: Run tests + run: | + pytest tests/ -v --tb=short --cov=rag_evaluation --cov-report=term-missing --cov-report=xml + + - name: Upload coverage report + if: matrix.python-version == '3.11' + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: coverage.xml + + test-docgpt: + name: Test DocGPT + runs-on: ubuntu-latest + defaults: + run: + working-directory: systems/docgpt + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Install dependencies + run: uv sync --dev + + - name: Run DocGPT tests + run: uv run pytest tests/ -v --tb=short || echo "No tests found yet" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..8c1afa1 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: CodeQL Security Analysis + +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + schedule: + # Run weekly on Monday at 8:00 UTC + - cron: "0 8 * * 1" + +permissions: + security-events: write + contents: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: [python] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + queries: +security-extended + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000..2a9a72c --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,24 @@ +name: Dependency Review + +on: + pull_request: + branches: [main, master] + +permissions: + contents: read + pull-requests: write + +jobs: + dependency-review: + name: Review Dependencies + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Dependency Review + uses: actions/dependency-review-action@v4 + with: + fail-on-severity: high + comment-summary-in-pr: always + deny-licenses: GPL-3.0, AGPL-3.0 diff --git a/.github/workflows/pr-changelog.yml b/.github/workflows/pr-changelog.yml new file mode 100644 index 0000000..7fc2a5f --- /dev/null +++ b/.github/workflows/pr-changelog.yml @@ -0,0 +1,136 @@ +name: PR Change Log + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + pull-requests: write + contents: read + +jobs: + log-changes: + name: Log PR Changes + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get PR diff stats + id: diff_stats + run: | + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + + # File change summary + echo "## Changed Files Summary" > pr_changes.md + echo "" >> pr_changes.md + + # Overall stats + STATS=$(git diff --shortstat $BASE_SHA...$HEAD_SHA) + echo "**Overall:** $STATS" >> pr_changes.md + echo "" >> pr_changes.md + + # Files changed with status + echo "### Files Changed" >> pr_changes.md + echo "" >> pr_changes.md + echo '| Status | File | Lines Added | Lines Removed |' >> pr_changes.md + echo '|--------|------|-------------|---------------|' >> pr_changes.md + + git diff --numstat --diff-filter=ACDMRT $BASE_SHA...$HEAD_SHA | while read added removed file; do + # Determine status + STATUS=$(git diff --name-status $BASE_SHA...$HEAD_SHA -- "$file" | head -1 | cut -f1) + case $STATUS in + A) STATUS_LABEL="Added" ;; + M) STATUS_LABEL="Modified" ;; + D) STATUS_LABEL="Deleted" ;; + R*) STATUS_LABEL="Renamed" ;; + C*) STATUS_LABEL="Copied" ;; + T) STATUS_LABEL="Type Changed" ;; + *) STATUS_LABEL="Changed" ;; + esac + + # Handle binary files + if [ "$added" = "-" ]; then + added="binary" + removed="binary" + fi + + echo "| $STATUS_LABEL | \`$file\` | +$added | -$removed |" >> pr_changes.md + done + + echo "" >> pr_changes.md + + # Breakdown by directory + echo "### Changes by Directory" >> pr_changes.md + echo "" >> pr_changes.md + git diff --stat $BASE_SHA...$HEAD_SHA | grep -E '^\s' | sed 's/|.*//' | xargs -I{} dirname {} | sort | uniq -c | sort -rn | while read count dir; do + echo "- **$dir/**: $count file(s)" >> pr_changes.md + done || true + + echo "" >> pr_changes.md + + # New dependencies check + if git diff $BASE_SHA...$HEAD_SHA -- requirements.txt pyproject.toml setup.py setup.cfg | grep -q '^+'; then + echo "### Dependency Changes" >> pr_changes.md + echo "" >> pr_changes.md + echo '```diff' >> pr_changes.md + git diff $BASE_SHA...$HEAD_SHA -- requirements.txt pyproject.toml setup.py setup.cfg || true + echo '```' >> pr_changes.md + echo "" >> pr_changes.md + fi + + # Large file warnings + echo "### Warnings" >> pr_changes.md + echo "" >> pr_changes.md + LARGE_FILES=$(git diff --numstat $BASE_SHA...$HEAD_SHA | awk '$1 != "-" && ($1+$2) > 300 {print $3 " (" $1 "+" " / " $2 "-)" }') + if [ -n "$LARGE_FILES" ]; then + echo "Large changes detected in:" >> pr_changes.md + echo "$LARGE_FILES" | while read line; do + echo "- \`$line\`" >> pr_changes.md + done + else + echo "No large file changes detected." >> pr_changes.md + fi + + # Save as output + BODY=$(cat pr_changes.md) + echo "changes<> $GITHUB_OUTPUT + echo "$BODY" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Comment on PR + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('pr_changes.md', 'utf8'); + + // Look for existing bot comment to update + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const marker = ''; + const fullBody = `${marker}\n# PR Change Log\n\n${body}\n\n---\n*Updated: ${new Date().toISOString()}*`; + + const existingComment = comments.find(c => c.body.includes(marker)); + if (existingComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body: fullBody, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: fullBody, + }); + } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..2b0bac8 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,45 @@ +name: Stale Issues & PRs + +on: + schedule: + # Run daily at 1:00 UTC + - cron: "0 1 * * *" + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + name: Close Stale Issues and PRs + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + # Issues + stale-issue-message: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed in 7 days if no further activity occurs. + Thank you for your contributions. + close-issue-message: > + This issue was closed because it has been stale for 7 days with no activity. + Feel free to reopen if this is still relevant. + days-before-issue-stale: 60 + days-before-issue-close: 7 + stale-issue-label: "stale" + exempt-issue-labels: "pinned,security,bug,enhancement" + + # Pull Requests + stale-pr-message: > + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed in 14 days if no further activity occurs. + close-pr-message: > + This pull request was closed because it has been stale for 14 days with no activity. + Feel free to reopen if you'd like to continue working on it. + days-before-pr-stale: 30 + days-before-pr-close: 14 + stale-pr-label: "stale" + exempt-pr-labels: "pinned,work-in-progress" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8a07a76 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,69 @@ +[project] +name = "rag-evaluation" +version = "0.1.0" +description = "A framework for evaluating Retrieval-Augmented Generation (RAG) models" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } + +dependencies = [ + "ragas>=0.1.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-cov>=4.0", + "ruff>=0.4.0", + "mypy>=1.0", +] +excel = [ + "openpyxl>=3.0.0", +] +bibtex = [ + "bibtexparser>=1.4.0", +] + +# ── Ruff ──────────────────────────────────────────────────────────── +[tool.ruff] +target-version = "py310" +line-length = 120 + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "N", # pep8-naming + "UP", # pyupgrade + "B", # flake8-bugbear + "SIM", # flake8-simplify + "RUF", # ruff-specific rules +] +ignore = [ + "E501", # line too long (handled by formatter) + "B008", # do not perform function calls in argument defaults +] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["S101"] # allow assert in tests +"examples/**" = ["E402", "F841"] # examples can have late imports and unused vars + +[tool.ruff.lint.isort] +known-first-party = ["rag_evaluation"] + +# ── Mypy ──────────────────────────────────────────────────────────── +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +ignore_missing_imports = true +check_untyped_defs = true + +# ── Pytest ────────────────────────────────────────────────────────── +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_functions = ["test_*"] +addopts = "-v --tb=short" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d354217 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,65 @@ +""" +Shared fixtures for RAG evaluation tests. +""" + +import pytest + +from rag_evaluation import RAGEvaluator + + +@pytest.fixture +def evaluator(): + """Create a default RAGEvaluator instance with all metrics.""" + return RAGEvaluator() + + +@pytest.fixture +def evaluator_faithfulness_only(): + """Create a RAGEvaluator with only faithfulness metric.""" + return RAGEvaluator(metrics=["faithfulness"]) + + +@pytest.fixture +def sample_data(): + """Sample evaluation data for testing.""" + return { + "query": "What is machine learning?", + "context": ( + "Machine learning is a subset of artificial intelligence that " + "enables systems to learn and improve from experience without " + "being explicitly programmed. It focuses on developing computer " + "programs that can access data and use it to learn for themselves." + ), + "answer": ( + "Machine learning is a subset of artificial intelligence. " + "It allows systems to learn from experience without explicit programming." + ), + "ground_truth": ( + "Machine learning is a subset of artificial intelligence that " + "enables systems to learn and improve from experience without " + "being explicitly programmed." + ), + } + + +@pytest.fixture +def batch_data(): + """Batch sample data for testing evaluate_batch.""" + return { + "queries": [ + "What is machine learning?", + "What is deep learning?", + ], + "contexts": [ + "Machine learning is a subset of AI that learns from data.", + "Deep learning uses neural networks with many layers to learn from large amounts of data.", + ], + "answers": [ + "Machine learning is a subset of AI.", + "Deep learning is a type of machine learning using neural networks.", + ], + "ground_truths": [ + "Machine learning is a subset of AI that learns from data.", + "Deep learning uses multi-layered neural networks.", + ], + } diff --git a/tests/test_data_ingestion.py b/tests/test_data_ingestion.py new file mode 100644 index 0000000..634b1dc --- /dev/null +++ b/tests/test_data_ingestion.py @@ -0,0 +1,94 @@ +""" +Tests for data ingestion loaders. +""" + +import os +import tempfile + +import pytest + +from rag_evaluation.data_ingestion import DataTableLoader + + +@pytest.fixture +def loader(): + return DataTableLoader() + + +@pytest.fixture +def csv_file(tmp_path): + """Create a temp CSV file that works cross-platform.""" + filepath = tmp_path / "test_data.csv" + filepath.write_text("query,context,answer\nWhat is AI?,AI is intelligence.,AI is smart.\n", encoding="utf-8") + return str(filepath) + + +@pytest.fixture +def json_file(tmp_path): + """Create a temp JSON array file.""" + filepath = tmp_path / "test_data.json" + filepath.write_text( + '[{"query": "What is AI?", "context": "AI is intelligence.", "answer": "AI is smart."}]', + encoding="utf-8", + ) + return str(filepath) + + +@pytest.fixture +def json_single_file(tmp_path): + """Create a temp JSON single-object file.""" + filepath = tmp_path / "test_single.json" + filepath.write_text( + '{"query": "What is AI?", "context": "AI is intelligence.", "answer": "AI is smart."}', + encoding="utf-8", + ) + return str(filepath) + + +@pytest.fixture +def eval_csv_file(tmp_path): + """Create a temp CSV with ground_truth column.""" + filepath = tmp_path / "test_eval.csv" + filepath.write_text( + "query,context,answer,ground_truth\nWhat is AI?,AI info.,AI is smart.,AI is intelligence.\n", + encoding="utf-8", + ) + return str(filepath) + + +class TestDataTableLoader: + """Tests for DataTableLoader CSV/JSON loading.""" + + def test_load_csv(self, loader, csv_file): + data = loader.load(csv_file) + assert data is not None + assert len(data) == 1 + assert data[0]["query"] == "What is AI?" + + def test_load_json(self, loader, json_file): + data = loader.load(json_file) + assert data is not None + assert len(data) == 1 + assert data[0]["query"] == "What is AI?" + + def test_load_json_single_object(self, loader, json_single_file): + data = loader.load(json_single_file) + assert len(data) == 1 + + def test_load_for_evaluation(self, loader, eval_csv_file): + data = loader.load_for_evaluation(eval_csv_file) + assert "queries" in data + assert "contexts" in data + assert "answers" in data + assert "ground_truths" in data + assert len(data["queries"]) == 1 + + def test_file_not_found(self, loader): + with pytest.raises(FileNotFoundError): + loader.load("/nonexistent/file.csv") + + def test_unsupported_format(self, loader, tmp_path): + filepath = tmp_path / "test.xyz" + filepath.write_text("data", encoding="utf-8") + with pytest.raises(ValueError): + loader.load(str(filepath)) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py new file mode 100644 index 0000000..64b4e9e --- /dev/null +++ b/tests/test_evaluator.py @@ -0,0 +1,126 @@ +""" +Tests for the RAGEvaluator class. +""" + +from rag_evaluation import RAGEvaluator + + +class TestRAGEvaluatorInit: + """Tests for RAGEvaluator initialization.""" + + def test_default_init_has_all_metrics(self, evaluator): + assert "faithfulness" in evaluator.metrics + assert "context_precision" in evaluator.metrics + assert "relevance" in evaluator.metrics + + def test_custom_metrics(self): + evaluator = RAGEvaluator(metrics=["faithfulness"]) + assert "faithfulness" in evaluator.metrics + assert "context_precision" not in evaluator.metrics + assert "relevance" not in evaluator.metrics + + def test_empty_metrics_list(self): + evaluator = RAGEvaluator(metrics=[]) + assert len(evaluator.metrics) == 0 + + def test_invalid_metric_ignored(self): + evaluator = RAGEvaluator(metrics=["nonexistent"]) + assert len(evaluator.metrics) == 0 + + +class TestRAGEvaluatorEvaluate: + """Tests for single evaluation.""" + + def test_evaluate_returns_all_metrics(self, evaluator, sample_data): + results = evaluator.evaluate(**sample_data) + assert "faithfulness" in results + assert "context_precision" in results + assert "relevance" in results + + def test_evaluate_faithfulness_has_score(self, evaluator, sample_data): + results = evaluator.evaluate(**sample_data) + assert "score" in results["faithfulness"] + assert isinstance(results["faithfulness"]["score"], float) + assert 0.0 <= results["faithfulness"]["score"] <= 1.0 + + def test_evaluate_without_ground_truth(self, evaluator, sample_data): + del sample_data["ground_truth"] + results = evaluator.evaluate(**sample_data) + assert results["context_precision"]["score"] is None + assert "error" in results["context_precision"] + + def test_evaluate_with_ground_truth(self, evaluator, sample_data): + results = evaluator.evaluate(**sample_data) + assert results["context_precision"]["score"] is not None + + def test_evaluate_relevance_has_score(self, evaluator, sample_data): + results = evaluator.evaluate(**sample_data) + assert "score" in results["relevance"] + assert isinstance(results["relevance"]["score"], float) + + def test_evaluate_empty_answer(self, evaluator, sample_data): + sample_data["answer"] = "" + results = evaluator.evaluate(**sample_data) + # Should handle gracefully without errors + assert "faithfulness" in results + + def test_evaluate_single_metric(self, evaluator_faithfulness_only, sample_data): + results = evaluator_faithfulness_only.evaluate(**sample_data) + assert "faithfulness" in results + assert "context_precision" not in results + assert "relevance" not in results + + +class TestRAGEvaluatorBatch: + """Tests for batch evaluation.""" + + def test_batch_returns_list(self, evaluator, batch_data): + results = evaluator.evaluate_batch(**batch_data) + assert isinstance(results, list) + assert len(results) == 2 + + def test_batch_each_result_has_metrics(self, evaluator, batch_data): + results = evaluator.evaluate_batch(**batch_data) + for result in results: + assert "faithfulness" in result + assert "context_precision" in result + assert "relevance" in result + + def test_batch_without_ground_truths(self, evaluator, batch_data): + del batch_data["ground_truths"] + results = evaluator.evaluate_batch( + queries=batch_data["queries"], + contexts=batch_data["contexts"], + answers=batch_data["answers"], + ) + assert len(results) == 2 + for result in results: + assert result["context_precision"]["score"] is None + + +class TestRAGEvaluatorAverageScores: + """Tests for average score computation.""" + + def test_average_scores_returns_dict(self, evaluator, batch_data): + results = evaluator.evaluate_batch(**batch_data) + averages = evaluator.get_average_scores(results) + assert isinstance(averages, dict) + + def test_average_scores_has_all_metrics(self, evaluator, batch_data): + results = evaluator.evaluate_batch(**batch_data) + averages = evaluator.get_average_scores(results) + assert "faithfulness" in averages + assert "context_precision" in averages + assert "relevance" in averages + + def test_average_scores_in_range(self, evaluator, batch_data): + results = evaluator.evaluate_batch(**batch_data) + averages = evaluator.get_average_scores(results) + for metric_name, score in averages.items(): + if score is not None: + assert 0.0 <= score <= 1.0, f"{metric_name} score out of range: {score}" + + def test_average_scores_empty_results(self, evaluator): + averages = evaluator.get_average_scores([]) + for score in averages.values(): + assert score is None diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..90c4ea9 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,86 @@ +""" +Tests for individual evaluation metrics. +""" + +from rag_evaluation.metrics.faithfulness import FaithfulnessMetric +from rag_evaluation.metrics.context_precision import ContextPrecisionMetric +from rag_evaluation.metrics.relevance import RelevanceMetric + + +class TestFaithfulnessMetric: + """Tests for the faithfulness metric.""" + + def setup_method(self): + self.metric = FaithfulnessMetric() + + def test_perfect_faithfulness(self): + context = "The sky is blue. Water is wet." + answer = "The sky is blue." + result = self.metric.compute(answer, context) + assert result["score"] >= 0.5 + + def test_empty_answer(self): + result = self.metric.compute("", "Some context") + assert result["score"] == 1.0 + assert result["details"]["total_sentences"] == 0 + + def test_result_has_details(self): + result = self.metric.compute("The sky is blue.", "The sky is blue.") + assert "details" in result + assert "total_sentences" in result["details"] + assert "supported_sentences" in result["details"] + + def test_score_in_range(self): + result = self.metric.compute( + "Machine learning is great for predictions.", + "Machine learning uses data to make predictions.", + ) + assert 0.0 <= result["score"] <= 1.0 + + +class TestContextPrecisionMetric: + """Tests for the context precision metric.""" + + def setup_method(self): + self.metric = ContextPrecisionMetric() + + def test_compute_returns_score(self): + result = self.metric.compute( + answer="ML is a subset of AI.", + context="ML is part of artificial intelligence.", + ground_truth="Machine learning is a subset of AI.", + ) + assert "score" in result + assert isinstance(result["score"], float) + + def test_score_in_range(self): + result = self.metric.compute( + answer="ML is AI.", + context="ML is part of AI.", + ground_truth="ML is AI.", + ) + assert 0.0 <= result["score"] <= 1.0 + + +class TestRelevanceMetric: + """Tests for the relevance metric.""" + + def setup_method(self): + self.metric = RelevanceMetric() + + def test_compute_returns_score(self): + result = self.metric.compute( + query="What is ML?", + answer="ML is a subset of AI.", + context="ML is part of artificial intelligence.", + ) + assert "score" in result + assert isinstance(result["score"], float) + + def test_score_in_range(self): + result = self.metric.compute( + query="What is ML?", + answer="ML is AI.", + context="ML is part of AI.", + ) + assert 0.0 <= result["score"] <= 1.0