diff --git a/.env.example b/.env.example index 67f33ba6..19cbdc8d 100644 --- a/.env.example +++ b/.env.example @@ -15,3 +15,6 @@ STORAGE_PATH=./storage # Optional: Custom API Base URL # OPENAI_API_BASE=your_custom_api_base_url + +# Optional: search backend URL (if using report_analyst_search_backend) +# SEARCH_BACKEND_URL=http://localhost:8001 diff --git a/.gitignore b/.gitignore index 95afcd34..821a1f31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ -/data +report_analyst/data .env *.log venv/ +venv2/ .gradio __pycache__/ @@ -21,6 +22,7 @@ report_analyst/temp/* # Coverage and test artifacts .coverage +coverage.xml .pytest_cache/ # Temporary files and diffs @@ -39,6 +41,9 @@ inputlucia # macOS system files .DS_Store +# Development notes and assessments (keep local, don't commit) +.devnotes/ + # Deployment guides and assessments (keep local, don't commit) CLOUDFLARE_ASSESSMENT.md CLOUDFLARE_QUICK_SUMMARY.md @@ -50,7 +55,14 @@ THEME_SWITCHING.md VERCEL_CHANGES_SUMMARY.md VERCEL_DEPLOYMENT_ASSESSMENT.md VERCEL_MIGRATION_GUIDE.md +.env +reports/ + +# Frontend generated artifacts +report_analyst_enterprise/components/web/node_modules/ +report_analyst_enterprise/components/web/dist/ +report_analyst_enterprise/components/streamlit_component/frontend/build/ node_modules Agents.md report-analyst.code-workspace -.vscode/launch.json \ No newline at end of file +.vscode/launch.json diff --git a/.gradio/certificate.pem b/.gradio/certificate.pem deleted file mode 100644 index b85c8037..00000000 --- a/.gradio/certificate.pem +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw -TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh -cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 -WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu -ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY -MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc -h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ -0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U -A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW -T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH -B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC -B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv -KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn -OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn -jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw -qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI -rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV -HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq -hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL -ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ -3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK -NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 -ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur -TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC -jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc -oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq -4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA -mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d -emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= ------END CERTIFICATE----- diff --git a/COLAB.md b/COLAB.md new file mode 100644 index 00000000..39daea13 --- /dev/null +++ b/COLAB.md @@ -0,0 +1,281 @@ +# Using report-analyst in Google Colab + +This guide explains how to use the **benchmarking and diagnostics** parts of +`report-analyst` from Google Colab (or any notebook environment). + +You can use it to: + +- **Evaluate retrieval systems from CSV files** (Precision@K, Recall@K, F1@K, + NDCG@K, MAP, MRR). +- **Generate detailed error-analysis tables** for retrieved chunks. +- **Align heterogeneous benchmark CSVs** (e.g. ClimRetrieve) to a common + internal schema using YAML configs. +- **Run PDF chunk subset diagnostics** to understand how different chunk sizes + relate to each other. + +The examples below focus on **offline evaluation and diagnostics**. They do not +run the Streamlit UI or the full backend. + +## 1. Installation in Colab + +Install directly from GitHub. For a clean install (e.g. after changing branches), you can uninstall and clear the pip cache first: + +```python +!pip uninstall report-analyst -y +!pip cache purge +``` + +Then install from your repo and branch (replace ``, ``, and `` with your values): + +```python +# Public repo – no token needed +!pip install --no-cache-dir "git+https://github.com//.git@#subdirectory=report-analyst" +``` + +If the repo is **private**, use a GitHub personal access token. Store it in Colab secrets or as an environment variable (e.g. `GITHUB_TOKEN`) and do **not** commit it: + +```python +import os +token = os.environ.get("GITHUB_TOKEN", "") # or from Colab secrets +if token: + !pip install --no-cache-dir "git+https://{token}@github.com//.git@#subdirectory=report-analyst" +else: + !pip install --no-cache-dir "git+https://github.com//.git@#subdirectory=report-analyst" +``` + +If the package lives at the repo root (no `report-analyst` subdirectory), omit `#subdirectory=report-analyst`. + +Optional boilerplate imports: + +```python +import pandas as pd +from pathlib import Path + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.core.benchmark.retrieval_results_loader import load_flexible_dataset_from_csv +from report_analyst.core.benchmark import error_analysis +from report_analyst.core.benchmark.dataset_mapper import ( + DatasetMapperFactory, + list_available_dataset_ids, +) +``` + +If your CSVs are on Google Drive, mount it first: + +```python +from google.colab import drive +drive.mount("/content/drive") +``` + +### Loading datasets from a Git repository + +You can load ground truth and benchmark CSVs from raw GitHub URLs. Use **raw** URLs (e.g. `raw.githubusercontent.com`), not blob/view URLs, so you get CSV content, not HTML. + +**Public repo:** + +```python +import requests + +GROUND_TRUTH_URL = "https://raw.githubusercontent.com////path/to/your_ground_truth.csv" +BENCHMARK_URL = "https://raw.githubusercontent.com////path/to/your_benchmark.csv" + +ground_truth_csv = requests.get(GROUND_TRUTH_URL).text +benchmark_csv = requests.get(BENCHMARK_URL).text +``` + +**Private repo:** pass your token in the `Authorization` header (use Colab secrets or env, never hardcode): + +```python +import os +import requests + +token = os.environ.get("GITHUB_TOKEN", "") # set in Colab environment or secrets +headers = {"Authorization": f"token {token}"} if token else {} + +ground_truth_csv = requests.get(GROUND_TRUTH_URL, headers=headers).text +benchmark_csv = requests.get(BENCHMARK_URL, headers=headers).text +``` + +Then pass these **strings** to `load_flexible_dataset_from_csv` using `csv_content=`, as in section 2. + +**Alternative – clone the repo:** if you prefer a local path, clone the repo (with token in the URL for private repos) and pass file paths to the loader instead of `csv_content`. + +## 2. Core workflow: evaluate retrieval from CSVs + +Load ground truth and benchmark from CSV **content** (strings from URLs or file reads), then evaluate with the flexible-dataset API. Use `compare_flexible_datasets`, not `compare_datasets`. + +```python +from pathlib import Path +import pandas as pd + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.core.benchmark.retrieval_results_loader import load_flexible_dataset_from_csv + +# If you loaded CSV strings from URLs (section 1), use csv_content= +ground_truth_ds = load_flexible_dataset_from_csv( + csv_content=ground_truth_csv, + dataset_name="ground_truth", +) +benchmark_ds = load_flexible_dataset_from_csv( + csv_content=benchmark_csv, + dataset_name="benchmark", +) + +# If you have local paths instead: +# ground_truth_ds = load_flexible_dataset_from_csv(csv_path="ground_truth_aligned.csv", dataset_name="ground_truth") +# benchmark_ds = load_flexible_dataset_from_csv(csv_path="benchmark_aligned.csv", dataset_name="benchmark") + +engine = EvaluationEngine() +k_values = [1, 3, 5, 10] +metrics = engine.compare_flexible_datasets(ground_truth_ds, benchmark_ds, k_values=k_values) + +# Build a metrics table (pivot) +rows = [] +for k, v in metrics.precision_at_k.items(): + rows.append({"k": k, "metric": "precision", "value": v}) +for k, v in metrics.recall_at_k.items(): + rows.append({"k": k, "metric": "recall", "value": v}) +for k, v in metrics.f1_at_k.items(): + rows.append({"k": k, "metric": "f1", "value": v}) +for k, v in metrics.ndcg_at_k.items(): + rows.append({"k": k, "metric": "ndcg", "value": v}) +rows.append({"k": None, "metric": "MAP", "value": metrics.mean_average_precision}) +rows.append({"k": None, "metric": "MRR", "value": metrics.mean_reciprocal_rank}) + +metrics_df = pd.DataFrame(rows) +metrics_df.pivot(index="metric", columns="k", values="value").round(4) +``` + +## 3. Error analysis in notebooks + +You can build a detailed, per-chunk error-analysis table similar to what the +Streamlit app exports, but directly inside a notebook. + +Use the flexible version of the helper so you can pass the same +`BenchmarkDataset` objects used above: + +```python +from report_analyst.core.benchmark import error_analysis + +top_k = 10 +df_error = error_analysis.build_error_analysis_dataframe_from_flexible( + ground_truth_dataset=ground_truth_ds, + benchmark_dataset=benchmark_ds, + top_k=top_k, +) + +# Inspect the first rows +display(df_error.head()) + +# Example: show only non-relevant retrieved chunks for manual inspection +df_false_positives = df_error[df_error["is_really_relevant"] == False] +display(df_false_positives.head()) +``` + +Columns in `df_error` include (depending on your data): + +- `report_name`, `question_id`, `question` +- `relevant_part_text` (from ground truth) +- `retrieved_chunk_text` (from benchmark) +- `position_in_top_k`, `model_score` +- `expert_relevance_label`, `is_really_relevant` +- `chunk_id`, `query_id` (identifiers) + +## 4. Dataset alignment via DatasetMapper + +If your raw CSVs do not yet match the internal benchmark schema, you can align +them using the same mapping logic that powers the CLI and Streamlit UI. + +Each dataset has a YAML config under `report_analyst/config/datasets/`. For +example, `climretrieve.yaml` defines how to map the ClimRetrieve CSVs. + +```python +import pandas as pd +from report_analyst.core.benchmark.dataset_mapper import ( + DatasetMapperFactory, + list_available_dataset_ids, +) + +print("Available dataset mapping IDs:", list_available_dataset_ids()) + +dataset_id = "climretrieve" # or another ID present in config/datasets +mapper = DatasetMapperFactory.get_mapper(dataset_id) + +# Raw CSVs from the original benchmark +df_gt_raw = pd.read_csv("climretrieve_ground_truth.csv") +df_bm_raw = pd.read_csv("climretrieve_benchmark.csv") + +# Align to the internal schema +df_gt_aligned = mapper.align_ground_truth(df_gt_raw) +df_bm_aligned = mapper.align_benchmark(df_bm_raw) + +display(df_gt_aligned.head()) +display(df_bm_aligned.head()) + +# Optionally, save aligned CSVs for later reuse +df_gt_aligned.to_csv("ground_truth_aligned.csv", index=False) +df_bm_aligned.to_csv("benchmark_results_aligned.csv", index=False) +``` + +You can then feed the aligned CSVs into the evaluation workflow described in +section 2. + +## 5. Chunk subset analysis (optional diagnostic) + +You can inspect how different chunk sizes relate to each other for a given PDF, +using the same `SentenceSplitter` logic as the analyzer but without embeddings. +This is independent from the IR metrics in `EvaluationEngine`, but can help you +understand whether smaller chunks are strict refinements or often fully +contained inside larger chunks. + +### From Python + +```python +from report_analyst.core.benchmark.chunk_subset_analysis import ( + analyze_pdf_chunk_subsets, + analyze_multiple_pdfs_chunk_subsets, +) + +# Single PDF, subset ratios between sizes 250, 440, 770 +result = analyze_pdf_chunk_subsets( + pdf_path="my_report.pdf", + chunk_sizes=[250, 440, 770], + chunk_overlap=20, +) + +# result.summary is a pandas DataFrame with: +# chunk_size_small, chunk_size_large, num_small, num_large, +# num_small_subsets, subset_ratio +display(result.summary) + +# Multiple PDFs (aggregated statistics across all files) +multi_result = analyze_multiple_pdfs_chunk_subsets( + pdf_paths=["report1.pdf", "report2.pdf"], + chunk_sizes=[250, 440, 770], + chunk_overlap=20, +) +display(multi_result.summary) +``` + +## 6. Scripts and local usage + +Scripts in `scripts/` (for example `align_benchmark_datasets.py`, +`evaluate_benchmark_from_csv.py`, `find_missing_relevant_parts.py`) are **not** +installed as commands when you `pip install` the library. + +To run them directly you should: + +1. Clone the repository locally. +2. Create a virtual environment and install dependencies. +3. Run the scripts from the project root, for example: + + ```bash + python scripts/evaluate_benchmark_from_csv.py \ + --reference path/to/ground_truth_aligned.csv \ + --input path/to/benchmark_results_aligned.csv \ + --k-values 1 3 5 10 + ``` + +In Colab, it is usually more convenient to **call the underlying Python +functions directly** (as shown in the examples above) instead of invoking these +scripts via shell commands. diff --git a/EXPECTED_FILE_FORMATS.md b/EXPECTED_FILE_FORMATS.md new file mode 100644 index 00000000..e4418f0f --- /dev/null +++ b/EXPECTED_FILE_FORMATS.md @@ -0,0 +1,110 @@ +# Expected File Formats for Datasets + +This document describes the structure expected for **ground truth** and **benchmark** datasets when using the Benchmarking / Evaluate features. Alignment (CSV/Excel only) maps your columns to these conventions. + +--- + +## CSV and Excel (information retrieval) + +Used for both ground truth and benchmark. Column names are **case-insensitive**; the loader accepts common variants. + +### After alignment (evaluation engine expects) + +| Role | Required columns | Optional / accepted variants | +|------|------------------|------------------------------| +| **Query** | `query_id` | `question_id`, `qid`, `query` | +| **Chunk** | `chunk_id` | `chunk`, `cid` | +| **Rank** | `position` | `rank`, `order`, `pos` | +| **Score** | `score` | `relevance_score`, `confidence_score`, `similarity_score` | +| **Document** | — | `report_id`, `document_id`, `doc_id`, `report` | + +For **ground truth** alignment (e.g. ClimRetrieve mapper), your CSV/Excel should have: + +- **document** (or `report`) — document/report identifier +- **question** — question text +- **context** or **relevant** — chunk/relevant text +- **relevance_label** (or `Source Relevance Score`, `relevance`, `label`) — relevance score or label + +For **benchmark** alignment: + +- **report** (or `document`) — report identifier +- **question** — question text +- **paragraph** (or `chunk`) — retrieved paragraph text +- **relevant_text** (or `relevant`) — optional; used for matching to ground truth +- **position** / **number** — optional; rank or paragraph number +- **relevant_text_sim** — optional; similarity score for ranking +- **label** / **relevance** — optional; relevance label + +If your file already has `query_id`, `chunk_id`, `position`, and `score` (or the accepted variants), it can be used without alignment. Otherwise you can: + +1. **Use a preset mapping** — Choose a dataset config (e.g. ClimRetrieve) under **Dataset Alignment** (CSV or Excel only) to convert to the expected structure. +2. **Flexible alignment wizard** — In the **Datasets** tab, use **Flexible Dataset Alignment (Wizard)** to:\n - Select which columns represent the query/criteria (single description or document + question).\n - Choose the chunk text column and, optionally, expert-annotated relevant parts.\n - Choose one or more ground-truth label columns (e.g. `relevance`, `usefulness`).\n - Choose one or more prediction/similarity columns on the benchmark side (e.g. model scores, `relevant_text_sim`).\n\n The wizard produces aligned ground truth and benchmark tables with a unified schema (`query_id`, `chunk_id`, optional `relevant_part_id`, text fields, labels, and predictions) so that both **ranking** and **classification** metrics can be derived from the same aligned data.\n3. **Configure columns manually** — In the upload flow, you can still use **Or configure columns manually** to map your columns directly to `query_id`, `chunk_id`, `position`, `score`, and `document` when you only need the basic IR schema. This is lighter than the wizard but provides fewer guarantees about compatibility between ground truth and benchmark datasets. + +--- + +## YAML and JSON (benchmark content schema) + +Used for **benchmark datasets** that follow the internal content schema (e.g. question sets with ground truth chunks). **Not** used for alignment; alignment works only for CSV and Excel. + +### Expected top-level structure + +```yaml +dataset_id: string +name: string +description: string +version: string +question_set: string +questions: array +``` + +### Each item in `questions` + +```yaml +question_id: string +question_text: string +ground_truth_chunks: array +``` + +### Each item in `ground_truth_chunks` + +```yaml +chunk_id: string +relevance_score: number # 0.0 to 1.0 +is_evidence: boolean +evidence_order: number # optional +annotation_notes: string # optional +``` + +Example (YAML): + +```yaml +dataset_id: my-benchmark +name: My Benchmark +description: Optional description +version: "1.0" +question_set: default +questions: + - question_id: q1 + question_text: "What are the company's emissions targets?" + ground_truth_chunks: + - chunk_id: c1 + relevance_score: 1.0 + is_evidence: true + evidence_order: 1 +``` + +**Note:** Report-level YAML (e.g. ClimRetrieve with `reports` → `questions` → `paragraphs`) is handled by a separate loader and is converted to the same internal format when supported. + +--- + +## Summary + +| Format | Use case | Alignment | +|--------|----------|-----------| +| **CSV** | Ground truth or benchmark (IR) | Yes — use Dataset Alignment (preset), the **Flexible alignment wizard**, or **Configure columns manually** if your columns differ. | +| **Excel** | Ground truth or benchmark (IR) | Yes — same as CSV. | +| **YAML** | Benchmark (content schema) | No — must match the schema above. | +| **JSON** | Benchmark (content schema) | No — same structure as YAML. | + +- For **preset** column mapping (e.g. ClimRetrieve), see `report_analyst/config/datasets/climretrieve.yaml` and add configs under `report_analyst/config/datasets/` for other conventions. +- For **custom** column names (e.g. `description`, `chunk_text`, `relevance`, `usefulness`, model prediction columns), you can either use the **Flexible alignment wizard** (recommended when you want both ranking and classification metrics) or **Configure columns manually** in the Dataset Management upload flow when you only need the basic IR schema. diff --git a/README.md b/README.md index c2118f59..b903cec3 100644 --- a/README.md +++ b/README.md @@ -93,11 +93,118 @@ python3 -m streamlit run report_analyst/streamlit_app.py In the web UI you can: - Upload a PDF sustainability report. -- Select a question set (e.g. TCFD, Lucia, Everest). +- Select a question set (e.g. TCFD, Lucia, Everest, ClimRetrieve). - Run the analysis and view: - Answers - Evidence and sources - Gaps and uncertainties +- Use the **Benchmarking** page to evaluate retrieval or extraction results against reference datasets (upload CSVs, align formats, run metrics, view results). + +## Running With Docker + +If you prefer to run the app in Docker instead of installing Python locally: + +1. **Build the image** + +```bash +docker build -t report-analyst . +``` + +2. **Run the Streamlit app** + +Make sure you have a `.env` file (for example by copying `.env.example`), then: + +```bash +docker run --rm -p 8501:8501 --env-file .env report-analyst +``` + +Open `http://localhost:8501` in your browser. + +3. **(Optional) Use Docker Compose** + +```bash +docker compose up app +``` + +You can also run CLI tools inside the same image, for example: + +```bash +docker run --rm --env-file .env report-analyst python scripts/test_climretrieve_benchmark.py --help +# Or with Compose: +docker compose run --rm cli python scripts/test_climretrieve_benchmark.py --help +``` + +The Dockerfile is configured to use Python 3.12 and installs system dependencies +required by the PDF and benchmarking stack (`libpoppler-cpp-dev`, `pkg-config`, etc.). + +## Running Benchmarks + +The project includes support for benchmarking retrieval and extraction systems against reference datasets. You can run benchmarks **from the Streamlit app** or via **command-line scripts**. + +### Benchmarking in the Streamlit App + +In the web UI, open the **Benchmarking** page from the sidebar. It provides four tabs: + +| Tab | Functionality | +|-----|----------------| +| **Datasets** | Upload ground truth and benchmark datasets (CSV, Excel, YAML, JSON). List and manage stored datasets. **Dataset Alignment (presets):** CSV/Excel can be aligned via configurable mappers (`report_analyst/config/datasets/`, e.g. ClimRetrieve). YAML/JSON that don’t match the app schema (e.g. ClimRetrieve Report-Level) are **aligned automatically** when possible (list-of-records with document, question, paragraph/relevance). **Flexible alignment (wizard):** for custom CSV/Excel files with expert labels and model predictions, use the **Flexible Dataset Alignment (Wizard)** section to interactively map query/criteria, chunk text, relevant parts, and label/prediction columns into a unified schema. Expected structures for CSV, Excel, YAML, and JSON: see [Expected file formats](EXPECTED_FILE_FORMATS.md). | +| **Evaluate** | Select a reference (ground truth) and a benchmark dataset, set K values and evaluation name, and run the evaluation. Metrics are computed with the same engine as the CLI (Precision@K, Recall@K, F1@K, NDCG@K, MAP, MRR). | +| **Results** | View and compare past evaluations, filter by dataset or date, and inspect metrics and charts. When a benchmark dataset also contains `relevance` / `usefulness` labels and corresponding `*_score_*` columns, this tab additionally shows a **Classification calibration (relevance/usefulness)** panel with calibration metrics (ECE, Brier scores, ROC AUC, F1) and a 3-class classification report for a selected model. | +| **Annotate** | Attach human annotations to evaluation results for later analysis. | + +To use the classification calibration view, prepare a benchmark dataset that includes: + +- Label columns: `relevance` and/or `usefulness` (ordinal labels 0–2). +- Model score columns: `relevance_score_` / `usefulness_score_` (continuous scores from your model or a notebook, for example as in `report_analyst/core/benchmark/02_classification_calibration.py`). + +Upload this dataset as a **Benchmark Dataset**. In the **Results** tab, pick the dataset, choose the label type and one or more score columns, then run the calibration analysis to inspect reliability and per-class performance. + +### ClimRetrieve Benchmark (CLI) + +Run the ClimRetrieve benchmark to evaluate your retrieval system against expert-annotated datasets: + +```bash +# Activate virtual environment +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Run the benchmark (downloads datasets automatically) +python scripts/test_climretrieve_benchmark.py +``` + +This will: +1. Download ClimRetrieve datasets from GitHub to `data/climretrieve/` +2. Transform datasets to the required format +3. Compare reference (ground truth) vs input (your results) datasets +4. Display evaluation metrics (MAP, MRR, Precision@K, Recall@K, F1@K, NDCG@K) + +**Options:** +- `--skip-download`: Use existing downloaded datasets +- `--reference-path PATH`: Specify custom reference dataset path +- `--input-path PATH`: Specify custom input dataset path +- `--k-values 1 5 10 20`: Custom K values for metrics + +**Requirements:** +- `openpyxl` for Excel file support (install with `pip install openpyxl`) +- Internet connection for initial download + +### Generic CSV Evaluation + +For **any** pair of ground-truth and results CSVs (not only ClimRetrieve), use the flexible loader and evaluation engine: + +- **Script:** `scripts/evaluate_benchmark_from_csv.py` — load two CSVs, run evaluation, print metrics and optionally write JSON. +- **Loader:** `load_flexible_dataset_from_csv` auto-detects IR vs IE datasets and accepts common column names (`query_id`/`question_id`, `chunk_id`, `position`/`rank`, `score`, etc.). +- **Engine:** `EvaluationEngine.compare_flexible_datasets` computes Precision@K, Recall@K, F1@K, NDCG@K, MAP, MRR. + +Example: + +```bash +python -m scripts.evaluate_benchmark_from_csv \ + --reference path/to/ground_truth.csv \ + --input path/to/benchmark_results.csv \ + --k-values 1 3 5 10 +``` + +For expected CSV formats (IR vs IE), script options, and ClimRetrieve-specific details, see `scripts/README_CLIMRETRIEVE.md`. For using the same workflow in **Google Colab** (install from Git, load CSVs from URLs, run evaluation and error analysis), see `COLAB.md`. For more detailed setup options (API, search backend, jobs), see `INSTALL.md`. @@ -126,7 +233,7 @@ Open Sustainability Analyst is used by various organizations and research instit | Feature | Description | |---------|-------------| -| Preset Question Sets | Use research-validated frameworks (TCFD, Lucia, Everest, Denali, Kilimanjaro) | +| Preset Question Sets | Use research-validated frameworks (TCFD, Lucia, Everest, ClimRetrieve, Denali, Kilimanjaro) | | Custom Question Selection | Choose specific questions from any framework | | Framework Comparison | Analyze the same document with different frameworks side-by-side | | Framework Extensibility | Add your own question sets via YAML files | @@ -169,6 +276,16 @@ Open Sustainability Analyst is used by various organizations and research instit | REST API | Optional FastAPI integration for other systems | | Backend Integration | Optional enterprise S3+NATS upload and processing | +### Benchmarking + +| Feature | Description | +|---------|-------------| +| Streamlit Benchmarking UI | Upload ground truth and benchmark CSV/Excel/YAML/JSON, align datasets (e.g. ClimRetrieve), run evaluations, view results and charts, attach human annotations | +| Flexible CSV Loader | Auto-detect IR vs IE datasets; accept common column names (query_id, chunk_id, position, score, etc.) | +| Evaluation Metrics | Precision@K, Recall@K, F1@K, NDCG@K, MAP, MRR via `EvaluationEngine`; calibration and classification metrics (ECE, Brier, ROC AUC, F1, 3-class report) via `classification_calibration` | +| Dataset Alignment | **Preset mappers:** CSV/Excel can be mapped via YAML configs (e.g. `config/datasets/climretrieve.yaml`). **Flexible wizard:** CSV/Excel can be aligned through a guided wizard that defines `query_id`, `chunk_id`, optional relevant-part IDs, and ground-truth/prediction columns so that both ranking and classification metrics can be computed from the same aligned data. **YAML/JSON:** when the file does not match the app schema, alignment is attempted automatically (list-of-records with document, question, chunk, score). | +| Colab Support | Run evaluation and error analysis from Google Colab; see `COLAB.md` | + ### Advanced Features | Feature | Description | @@ -189,6 +306,8 @@ Current core question sets (in `report_analyst/questionsets/`): Comprehensive sustainability labeling and gap analysis framework (35+ questions). - **TCFD** – `tcfd_questions.yaml` Climate-related financial disclosure questions aligned with TCFD. +- **ClimRetrieve** – `climretrieve_questions.yaml` + Climate-focused questions aligned with the [ClimRetrieve](https://arxiv.org/abs/2406.09818) benchmark (adaptation, scenarios, targets, risks, etc.). Use this set when evaluating retrieval or analysis against ClimRetrieve datasets. - **Denali** – `denali_questions.yaml` Deeper sustainability analysis for specific focus areas. - **Kilimanjaro** – `kilimanjaro_questions.yaml` @@ -247,7 +366,10 @@ This repository is intentionally modular. The separation also reflects **differe report-analyst/ ├── report_analyst/ # Core open-source analysis engine (RPL) │ ├── core/ # Chunking, analysis, caching, workflows +│ │ └── benchmark/ # Evaluation engine, CSV loader, error analysis, dataset mapper +│ ├── config/datasets/ # YAML configs for dataset alignment (e.g. climretrieve.yaml) │ ├── questionsets/ # Question set YAML files (frameworks) +│ ├── ui/benchmarking.py # Streamlit benchmarking UI (datasets, evaluate, results, annotate) │ ├── streamlit_app.py # Main Streamlit application │ └── streamlit_app_backend.py # Legacy / backend-focused UI ├── report_analyst_api/ # FastAPI REST API (Climate+Tech Open License for Good) @@ -381,6 +503,8 @@ pytest tests/ -v --cov=report_analyst --cov-report=term-missing For detailed deployment patterns (Docker, Kubernetes, NATS workers, etc.), see: - `INSTALL.md` – Installation and configuration options +- `COLAB.md` – Using benchmarking and evaluation in Google Colab +- `scripts/README_CLIMRETRIEVE.md` – ClimRetrieve benchmark and generic CSV evaluation - `report_analyst_jobs/README.md` – Job processing and worker patterns - `docs/CI.md` – Continuous integration and testing diff --git a/docs/BENCHMARK_DATASET_FLOW.md b/docs/BENCHMARK_DATASET_FLOW.md new file mode 100644 index 00000000..503da85a --- /dev/null +++ b/docs/BENCHMARK_DATASET_FLOW.md @@ -0,0 +1,257 @@ +# Benchmark Dataset Flow + +This document describes the flexible dataset flow for benchmarking, supporting both Information Retrieval (IR) and Information Extraction (IE) datasets. + +## Overview + +The benchmarking system supports comparing two datasets with flexible column names: +- **Reference Dataset**: Ground truth (e.g., "climretrieve" for IR, "chatreport" for IE) +- **Input Dataset**: Actual results to evaluate + +### Dataset Types + +1. **Information Retrieval (IR)**: Compare retrieved chunks + - Example: climretrieve dataset + - Compares: chunk_id, position, relevance scores + +2. **Information Extraction (IE)**: Compare analysis/answers + - Example: chatreport, climatefinancebench datasets + - Compares: answers, categories, extracted values + +## Data Format + +The system is flexible and supports different column names. It automatically detects dataset type (IR vs IE) and maps columns. + +### Information Retrieval (IR) CSV Format + +```csv +query_id,report_id,chunk_id,chunk_text,position,score,similarity_score,llm_score +tcfd_1,report_001,chunk_001,"Climate risks include...",1,0.95,0.92,0.88 +tcfd_1,report_001,chunk_015,"Risk assessment...",2,0.89,0.87,0.85 +``` + +**Common column name variations (all supported):** +- Query ID: `query_id`, `question_id`, `qid`, `query` +- Chunk ID: `chunk_id`, `chunk`, `cid` +- Position: `position`, `rank`, `order`, `pos` +- Score: `score`, `relevance_score`, `confidence_score`, `similarity_score` +- Report ID: `report_id`, `document_id`, `doc_id`, `report` + +### Information Extraction (IE) CSV Format + +```csv +question_id,answer,category,confidence_score,extracted_value +tcfd_1,"The company identifies climate risks...","risk_identification",0.92,"High" +tcfd_2,"Strategy includes...","strategy",0.88,"Medium" +``` + +**Common column name variations (all supported):** +- Query ID: `query_id`, `question_id`, `qid`, `query` +- Answer: `answer`, `analysis`, `response`, `text` +- Category: `category`, `class`, `label`, `type` +- Score: `score`, `confidence_score`, `relevance_score` + +### SQLite Format + +Same structure as CSV, stored in a table (default: `retrieval_results`). + +## Flow A: Export Our Retrieval Results + +Export retrieval results from our internal tools to CSV/SQLite format. + +### Example: Export to CSV + +```python +from report_analyst.core.benchmark.retrieval_results_loader import export_retrieval_results_to_csv +from report_analyst.models.benchmark import RetrievalResultsDataset, RetrievalResultRow + +# Create dataset from our retrieval results +results = [ + RetrievalResultRow( + query_id="tcfd_1", + report_id="report_001", + chunk_id="chunk_001", + chunk_text="Climate risks...", + position=1, + score=0.95, + similarity_score=0.92 + ), + # ... more results +] + +dataset = RetrievalResultsDataset( + dataset_id="our_results_v1", + name="Our Retrieval Results", + source="internal", + results=results +) + +# Export to CSV +export_retrieval_results_to_csv(dataset, "our_results.csv") +``` + +## Flow B: Load User-Provided CSV (Webhook-Ready) + +Load retrieval results from CSV file or content. This function is designed to be webhook-ready. + +### Function: `load_retrieval_results_from_csv` + +```python +from report_analyst.core.benchmark.retrieval_results_loader import load_retrieval_results_from_csv + +# Option 1: Load from file path +dataset = load_retrieval_results_from_csv( + csv_path="user_results.csv", + dataset_id="user_results_v1", + dataset_name="User Uploaded Results" +) + +# Option 2: Load from CSV content (webhook-ready) +csv_content = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89""" + +dataset = load_retrieval_results_from_csv( + csv_content=csv_content, + dataset_id="webhook_results", + dataset_name="Webhook Results" +) +``` + +### Webhook Example + +```python +from fastapi import FastAPI, UploadFile, File +from report_analyst.core.benchmark.retrieval_results_loader import load_retrieval_results_from_csv + +app = FastAPI() + +@app.post("/api/benchmark/upload-results") +async def upload_retrieval_results(file: UploadFile = File(...)): + """Webhook endpoint to upload retrieval results CSV""" + csv_content = await file.read() + + dataset = load_retrieval_results_from_csv( + csv_content=csv_content, + dataset_id=f"upload_{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}", + dataset_name=file.filename + ) + + return {"dataset_id": dataset.dataset_id, "num_results": len(dataset.results)} +``` + +## Flow C: Load from SQLite + +```python +from report_analyst.core.benchmark.retrieval_results_loader import load_retrieval_results_from_sqlite + +dataset = load_retrieval_results_from_sqlite( + db_path="retrieval_results.db", + table_name="retrieval_results", + dataset_id="sqlite_results", + dataset_name="SQLite Results", + query_filter="report_id = 'report_001'" # Optional filter +) +``` + +## Comparing Datasets + +Compare a reference dataset (ground truth) against an input dataset (actual results). + +### Example + +```python +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.core.benchmark.retrieval_results_loader import ( + load_retrieval_results_from_csv, + load_retrieval_results_from_sqlite +) + +# Load reference dataset (e.g., climretrieve) +reference = load_retrieval_results_from_csv( + csv_path="climretrieve_reference.csv", + dataset_id="climretrieve_v1", + dataset_name="ClimRetrieve Reference" +) + +# Load input dataset (our results or user upload) +input_dataset = load_retrieval_results_from_csv( + csv_path="our_results.csv", + dataset_id="our_results_v1", + dataset_name="Our Results" +) + +# Compare datasets +engine = EvaluationEngine() +metrics = engine.compare_datasets( + reference_dataset=reference, + input_dataset=input_dataset, + k_values=[1, 3, 5, 10] +) + +print(f"MAP: {metrics.mean_average_precision:.3f}") +print(f"MRR: {metrics.mean_reciprocal_rank:.3f}") +print(f"Precision@5: {metrics.precision_at_k[5]:.3f}") +``` + +## Models + +### RetrievalResultRow + +Single row in a retrieval results dataset. + +```python +class RetrievalResultRow(BaseModel): + query_id: str + report_id: Optional[str] + chunk_id: str + chunk_text: Optional[str] + position: int # 1-indexed + score: float + similarity_score: Optional[float] + llm_score: Optional[float] + metadata: Dict[str, Any] +``` + +### RetrievalResultsDataset + +Collection of retrieval results. + +```python +class RetrievalResultsDataset(BaseModel): + dataset_id: str + name: str + description: Optional[str] + source: str # 'csv', 'sqlite', 'internal' + source_path: Optional[str] + results: List[RetrievalResultRow] + + def get_results_by_query(self, query_id: str) -> List[RetrievalResultRow] + def get_unique_queries(self) -> List[str] + def get_unique_reports(self) -> List[str] +``` + +## Key Functions + +### Loaders + +- `load_retrieval_results_from_csv()` - Load from CSV file or content (webhook-ready) +- `load_retrieval_results_from_sqlite()` - Load from SQLite database + +### Exporters + +- `export_retrieval_results_to_csv()` - Export dataset to CSV file + +### Evaluation + +- `EvaluationEngine.compare_datasets()` - Compare reference vs input datasets + +## Migration from Old Format + +The old format (YAML with ground truth chunks) is still supported through `BenchmarkDatasetContent`. The new format (`RetrievalResultsDataset`) is for comparing two retrieval result sets directly. + +## Next Steps + +1. Update UI to support CSV upload +2. Add export functionality to convert our retrieval results to CSV +3. Integrate with existing analyzer to export results automatically diff --git a/docs/benchmarking_plan.md b/docs/benchmarking_plan.md new file mode 100644 index 00000000..0d44954c --- /dev/null +++ b/docs/benchmarking_plan.md @@ -0,0 +1,283 @@ +# Information Retrieval Benchmarking Plan + +## Overview + +This document outlines the plan for implementing dataset upload capabilities, benchmarking the information retrieval pipeline, and adding human annotation functionality to the report-analyst system. + +## 1. Dataset Upload and Management + +### Data Structure +Benchmark datasets will be stored in YAML format: + +```yaml +dataset_id: "tcfd_climate_risks_v1" +name: "TCFD Climate Risks Benchmark Dataset" +description: "Ground truth chunk relevance for TCFD climate risk questions" +version: "1.0" +created_at: "2024-01-15" +question_set: "tcfd" +questions: + - question_id: "tcfd_1" + question_text: "What are the climate-related risks?" + ground_truth_chunks: + - chunk_id: "chunk_001" + relevance_score: 1.0 # Ground truth relevance (0.0-1.0) + is_evidence: true + evidence_order: 1 + annotation_notes: "Contains specific climate risk metrics" +``` + +### Database Schema Extensions + +```sql +-- Benchmark datasets table +CREATE TABLE benchmark_datasets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dataset_id TEXT UNIQUE, + name TEXT, + description TEXT, + version TEXT, + question_set TEXT, + file_path TEXT, -- Path to the dataset file + created_at TIMESTAMP, + updated_at TIMESTAMP +); + +-- Ground truth chunk relevance +CREATE TABLE ground_truth_chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dataset_id TEXT, + question_id TEXT, + chunk_id TEXT, + relevance_score REAL, + is_evidence BOOLEAN, + evidence_order INTEGER, + annotation_notes TEXT, + created_at TIMESTAMP, + FOREIGN KEY(dataset_id) REFERENCES benchmark_datasets(dataset_id) +); + +-- Benchmark evaluation results +CREATE TABLE benchmark_evaluations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dataset_id TEXT, + evaluation_name TEXT, + config_hash TEXT, -- Hash of retrieval configuration + retrieval_config TEXT, -- JSON of retrieval parameters + evaluation_metrics TEXT, -- JSON of computed metrics + created_at TIMESTAMP, + FOREIGN KEY(dataset_id) REFERENCES benchmark_datasets(dataset_id) +); + +-- Human annotations table +CREATE TABLE human_annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + evaluation_id INTEGER, + question_id TEXT, + chunk_id TEXT, + human_relevance_score REAL, + human_is_evidence BOOLEAN, + human_evidence_order INTEGER, + annotation_notes TEXT, + annotator_id TEXT, + created_at TIMESTAMP, + FOREIGN KEY(evaluation_id) REFERENCES benchmark_evaluations(id) +); +``` + +## 2. Benchmarking Engine + +### Core Components + +1. **Dataset Loader**: Parse and validate benchmark datasets +2. **Evaluation Engine**: Compare retrieval results against ground truth +3. **Metrics Calculator**: Compute various evaluation metrics +4. **Configuration Manager**: Track different retrieval configurations + +### Evaluation Metrics + +- **Precision@K**: Percentage of retrieved chunks that are relevant +- **Recall@K**: Percentage of relevant chunks that were retrieved +- **F1@K**: Harmonic mean of precision and recall +- **Mean Reciprocal Rank (MRR)**: Average of reciprocal ranks of first relevant chunk +- **Normalized Discounted Cumulative Gain (NDCG)**: Considers relevance scores and ranking +- **Mean Average Precision (MAP)**: Average precision across all questions + +### Configuration Management + +```python +class RetrievalConfig: + chunk_size: int + chunk_overlap: int + top_k: int + use_llm_scoring: bool + embedding_model: str + similarity_threshold: float + llm_model: str +``` + +## 3. Human Annotation Interface + +### Annotation Workflow + +1. **Batch Annotation**: Annotate multiple chunks at once +2. **Individual Annotation**: Fine-tune individual chunk scores +3. **Evidence Assignment**: Mark chunks as evidence and assign order +4. **Notes and Comments**: Add contextual notes for annotations + +### Interface Features + +- Side-by-side comparison of retrieved vs. ground truth chunks +- Slider controls for relevance scores (0.0-1.0) +- Checkbox for evidence assignment +- Text areas for annotation notes +- Bulk operations for similar chunks +- Export/import annotation data + +## 4. UI Integration + +### New Streamlit Pages + +1. **Dataset Management Page** + - Upload benchmark datasets + - View existing datasets + - Dataset validation and preview + - Dataset versioning + +2. **Benchmarking Page** + - Select dataset and retrieval configuration + - Run benchmark evaluations + - View evaluation results and metrics + - Compare different configurations + - Export evaluation reports + +3. **Annotation Page** + - Select evaluation results to annotate + - Interactive annotation interface + - Progress tracking + - Annotation quality metrics + +4. **Results Dashboard** + - Historical evaluation trends + - Configuration performance comparison + - Annotation statistics + - Model improvement tracking + +## 5. API Extensions + +### New Endpoints + +```python +# Dataset management +POST /api/datasets/upload +GET /api/datasets +GET /api/datasets/{dataset_id} +DELETE /api/datasets/{dataset_id} + +# Benchmarking +POST /api/benchmarks/evaluate +GET /api/benchmarks/results +GET /api/benchmarks/{evaluation_id}/metrics + +# Annotations +POST /api/annotations/batch +GET /api/annotations/{evaluation_id} +PUT /api/annotations/{annotation_id} +``` + +## 6. File Structure + +``` +app/ +├── core/ +│ ├── benchmark/ +│ │ ├── __init__.py +│ │ ├── dataset_loader.py +│ │ ├── evaluation_engine.py +│ │ ├── metrics_calculator.py +│ │ └── config_manager.py +│ ├── annotation/ +│ │ ├── __init__.py +│ │ ├── annotation_manager.py +│ │ └── annotation_validator.py +│ └── storage/ +│ ├── benchmark_store.py +│ └── annotation_store.py +├── models/ +│ ├── benchmark.py +│ └── annotation.py +├── ui/ +│ ├── dataset_management.py +│ ├── benchmarking.py +│ └── annotation_interface.py +└── api/ + ├── benchmark_routes.py + └── annotation_routes.py +``` + +## 7. Implementation Phases + +### Phase 1: Core Infrastructure (Week 1-2) +- Database schema extensions +- Dataset loader and validator +- Basic evaluation engine +- Core metrics calculation + +### Phase 2: Benchmarking Engine (Week 3-4) +- Configuration management +- Evaluation workflow +- Results storage and retrieval +- Basic UI for running benchmarks + +### Phase 3: Annotation System (Week 5-6) +- Annotation data model +- Annotation interface +- Batch annotation capabilities +- Annotation validation + +### Phase 4: UI Integration (Week 7-8) +- Streamlit page integration +- Results visualization +- Configuration comparison +- Export/import functionality + +### Phase 5: Advanced Features (Week 9-10) +- Advanced metrics (NDCG, MAP) +- Annotation quality assessment +- Automated improvement suggestions +- Performance optimization + +## 8. Data Flow + +1. **Dataset Upload**: User uploads benchmark dataset → Validation → Storage +2. **Benchmark Execution**: Select dataset + config → Run retrieval → Compare with ground truth → Calculate metrics → Store results +3. **Annotation Process**: Select evaluation results → Annotate chunks → Store annotations → Update evaluation metrics +4. **Analysis**: View historical trends → Compare configurations → Identify improvement opportunities + +## 9. Configuration Management + +### Retrieval Configurations +- Predefined configurations (fast, balanced, accurate) +- Custom configuration builder +- Configuration templates +- A/B testing framework + +### Evaluation Settings +- Metrics to compute +- Evaluation thresholds +- Comparison baselines +- Export formats + +## 10. Quality Assurance + +### Dataset Validation +- Schema validation +- Data consistency checks +- Coverage analysis +- Duplicate detection + +### Annotation Quality +- Inter-annotator agreement +- Annotation consistency checks +- Quality metrics +- Automated validation \ No newline at end of file diff --git a/package.json b/package.json index f245b85f..3317d6bd 100644 --- a/package.json +++ b/package.json @@ -1 +1 @@ -{"name":"report-analyst","version":"0.8.0-rc","scripts":{"backend":"uvicorn app.main:app --reload --port 8000","frontend":"python app/gradio_app.py","dev":"pnpm run backend & pnpm run frontend","clean":"mkdir -p data/output data/input && rm -rf data/output/* && rm -f backend.log frontend.log"}} +{"name":"report-analyst","version":"0.8.0-rc","scripts":{"backend":"uvicorn app.main:app --reload --port 8000","clean":"mkdir -p data/output data/input && rm -rf data/output/* && rm -f backend.log frontend.log"}} diff --git a/pyproject.toml b/pyproject.toml index f22dafe3..fafdfc16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,31 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "report-analyst" +version = "0.8.0rc" +description = "Open Sustainability Analyst - benchmark evaluation, error analysis, and dataset loading for retrieval evaluation" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "pandas", + "numpy", + "pydantic", + "pyyaml", +] + +[tool.setuptools.packages.find] +where = ["."] +include = ["report_analyst", "report_analyst.*"] + +[tool.setuptools.package-data] +"report_analyst" = ["data/*.csv", "config/datasets/*.yaml"] + [tool.isort] profile = "black" line_length = 127 -skip = [".git", ".hg", "venv", "venv2", "node_modules", "__pycache__"] +skip = [".git", ".hg", "venv", "venv2", "node_modules", "report_analyst_enterprise", "__pycache__", "htmlcov", "*.egg-info", "report_analyst/core/benchmark/02_classification_calibration.py"] [tool.black] line-length = 127 @@ -10,13 +34,20 @@ exclude = ''' /( \.git | \.hg + | \.venv | venv | venv2 | node_modules + | report_analyst_enterprise | __pycache__ + | \.pytest_cache + | htmlcov )/ ''' +[tool.pytest.ini_options] +asyncio_default_fixture_loop_scope = "function" + [tool.ruff] target-version = "py312" line-length = 127 diff --git a/report_analyst/config/datasets/climretrieve.yaml b/report_analyst/config/datasets/climretrieve.yaml new file mode 100644 index 00000000..56edd7a5 --- /dev/null +++ b/report_analyst/config/datasets/climretrieve.yaml @@ -0,0 +1,19 @@ +id: climretrieve + +ground_truth: + columns: + document: document + question: question + context: context + relevant: relevant + page_number: page_number + relevance_label: "Source Relevance Score" + +benchmark: + columns: + report_id: report + question: question + paragraph: paragraph + relevant_text: relevant_text + relevance_label: label + diff --git a/report_analyst/core/analyzer.py b/report_analyst/core/analyzer.py index cef5ae6e..92c9cb45 100644 --- a/report_analyst/core/analyzer.py +++ b/report_analyst/core/analyzer.py @@ -2,25 +2,17 @@ import json import logging import os -import pickle import re -import shutil -import sqlite3 -import sys from pathlib import Path -from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple +from typing import Any, AsyncGenerator, Dict, List, Optional import numpy as np import pandas as pd import yaml from dotenv import load_dotenv -from langchain.chains import RetrievalQA -from langchain.chains.summarize import load_summarize_chain -from langchain.prompts import PromptTemplate -from langchain_openai import ChatOpenAI from llama_index.core import Document, Settings from llama_index.core.ingestion import IngestionCache -from llama_index.core.llms import ChatMessage, MessageRole +from llama_index.core.llms import ChatMessage from llama_index.core.node_parser import SentenceSplitter from llama_index.embeddings.openai import OpenAIEmbedding from llama_index.readers.file import PyMuPDFReader @@ -50,7 +42,10 @@ # Log available model keys logger.info(f"API Keys available - OpenAI: {bool(openai_key)}, Gemini: {bool(gemini_key)}") logger.info( - f"Backend mode - USE_BACKEND: {use_backend}, USE_CENTRALIZED_LLM: {use_centralized_llm}, USE_FULL_BACKEND_ANALYSIS: {use_full_backend_analysis}" + "Backend mode - USE_BACKEND: %s, USE_CENTRALIZED_LLM: %s, USE_FULL_BACKEND_ANALYSIS: %s", + use_backend, + use_centralized_llm, + use_full_backend_analysis, ) # If using backend for LLM, don't require local API keys @@ -137,10 +132,6 @@ def __init__(self): self.cache_path.mkdir(parents=True, exist_ok=True) self.llm_cache_path.mkdir(parents=True, exist_ok=True) - log_analysis_step(f"Storage path: {self.storage_path.resolve()}", "debug") - log_analysis_step(f"Cache path: {self.cache_path.resolve()}", "debug") - log_analysis_step(f"LLM cache path: {self.llm_cache_path.resolve()}", "debug") - # Set default question set self.question_set = "tcfd" self.questions = self._load_questions() @@ -153,10 +144,6 @@ def __init__(self): self.use_backend_llm = use_backend and (use_centralized_llm or use_full_backend_analysis) if self.use_backend_llm: - log_analysis_step( - "Skipping local LLM initialization - using backend for all LLM functionality", - "info", - ) # Set minimal placeholders for compatibility self.llm = None self.embeddings = None @@ -230,11 +217,11 @@ def _initialize_llm_clients(self) -> None: else: logger.warning("No OpenAI API key - embedding functionality will be limited") except ValueError as e: - log_analysis_step(f"LLM initialization deferred: {str(e)}", "warning") + log_analysis_step(f"LLM initialization deferred: {e!s}", "warning") self.llm = None self.embeddings = None except Exception as e: - log_analysis_step(f"Error initializing local LLM clients: {str(e)}", "error") + log_analysis_step(f"Error initializing local LLM clients: {e!s}", "error") raise def _get_cache_key(self, file_path: str) -> str: @@ -275,7 +262,7 @@ def _get_cache_key(self, file_path: str) -> str: else: # Local file path (existing behavior - maintain backwards compatibility) return f"{Path(file_path).stem}_{params_str}" - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning(f"[ANALYSIS] Cache ERROR: Failed to generate cache key: {e}") # Fallback: try to extract identifier safely if file_path.startswith("urn:report-analyst:backend:"): @@ -284,8 +271,8 @@ def _get_cache_key(self, file_path: str) -> str: else: try: return f"{Path(file_path).stem}_fallback" - except: - return f"unknown_fallback" + except Exception: # noqa: BLE001 + return "unknown_fallback" def _get_vector_store_collection_name(self, cache_key: str) -> str: """Generate a valid collection name from cache key.""" @@ -312,7 +299,7 @@ def _load_chunks_cache(self, cache_key: str) -> Optional[List]: return chunks logger.info("[ANALYSIS] Cache MISS: No cached chunks found") return None - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning(f"[ANALYSIS] Cache ERROR: Failed to load chunks cache: {e}") return None @@ -331,7 +318,7 @@ def _save_chunks_cache(self, cache_key: str, chunks: List) -> None: with open(cache_file, "w", encoding="utf-8") as f: json.dump(serializable_chunks, f) logger.info(f"[ANALYSIS] ✓ Cache SAVE: Saved {len(chunks)} chunks to cache") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning(f"[ANALYSIS] Cache ERROR: Failed to save chunks cache: {e}") def _load_vector_store(self, cache_key: str, chunks: List) -> Optional[LlamaVectorStore]: @@ -346,7 +333,7 @@ def _load_vector_store(self, cache_key: str, chunks: List) -> Optional[LlamaVect vector_store = LlamaVectorStore(store_dir) # Try to load the store - this will verify if it's valid if vector_store.load(): - logger.info(f"[ANALYSIS] ✓ Cache HIT: Loaded vector store from cache") + logger.info("[ANALYSIS] ✓ Cache HIT: Loaded vector store from cache") return vector_store except Exception as inner_e: logger.error( @@ -358,7 +345,7 @@ def _load_vector_store(self, cache_key: str, chunks: List) -> Optional[LlamaVect return None except Exception as e: logger.warning(f"[ANALYSIS] Cache ERROR: Failed to load vector store cache: {e}") - logger.debug(f"Full vector store cache error: {str(e)}", exc_info=True) + logger.debug(f"Full vector store cache error: {e!s}", exc_info=True) return None async def score_chunk_relevance(self, question: str, chunk_text: str) -> float: @@ -366,59 +353,51 @@ async def score_chunk_relevance(self, question: str, chunk_text: str) -> float: if not self.use_cache: Settings.ingestion_cache = None - log_analysis_step(f"Computing relevance score for chunk: {chunk_text[:100]}...") - try: response = await self.llm.achat( - prompt=f"""As a senior equity analyst with expertise in climate science evaluating a company's sustainability report, you are tasked with evaluating text fragments for their usefulness in answering specific TCFD questions. - -Your task is to score the relevance and quality of evidence in each text fragment. Consider: - -1. Specificity and Concreteness: - - Quantitative data and specific metrics (highest value) - - Concrete policies and procedures - - Specific commitments with timelines - - General statements or vague claims (lowest value) - -2. Evidence Quality: - - Verifiable data and third-party verification - - Clear methodologies and frameworks - - Specific examples and case studies - - Unsubstantiated claims (lowest value) - -3. Direct Relevance: - - Direct answers to the question components - - Related but indirect information - - Contextual background - - Unrelated information (lowest value) - -4. Disclosure Quality: - - Comprehensive and transparent disclosure - - Balanced reporting (both positive and negative) - - Clear acknowledgment of limitations - - Potential greenwashing or selective disclosure (lowest value) - -Score from 0.0 to 1.0 where: -0.0 = Not useful (generic statements, unrelated content) -0.3 = Contains relevant context but no specific evidence -0.6 = Contains useful specific information but requires additional context -1.0 = Contains critical evidence or specific details that directly answer the question - -Question: {question} - -Text to evaluate: -{chunk_text} - -Output only the numeric score (0.0-1.0):""" + prompt=( + "As a senior equity analyst with expertise in climate science evaluating a company's " + "sustainability report, you are tasked with evaluating text fragments for their " + "usefulness in answering specific TCFD questions.\n\n" + "Your task is to score the relevance and quality of evidence in each text fragment. Consider:\n\n" + "1. Specificity and Concreteness:\n" + " - Quantitative data and specific metrics (highest value)\n" + " - Concrete policies and procedures\n" + " - Specific commitments with timelines\n" + " - General statements or vague claims (lowest value)\n\n" + "2. Evidence Quality:\n" + " - Verifiable data and third-party verification\n" + " - Clear methodologies and frameworks\n" + " - Specific examples and case studies\n" + " - Unsubstantiated claims (lowest value)\n\n" + "3. Direct Relevance:\n" + " - Direct answers to the question components\n" + " - Related but indirect information\n" + " - Contextual background\n" + " - Unrelated information (lowest value)\n\n" + "4. Disclosure Quality:\n" + " - Comprehensive and transparent disclosure\n" + " - Balanced reporting (both positive and negative)\n" + " - Clear acknowledgment of limitations\n" + " - Potential greenwashing or selective disclosure (lowest value)\n\n" + "Score from 0.0 to 1.0 where:\n" + "0.0 = Not useful (generic statements, unrelated content)\n" + "0.3 = Contains relevant context but no specific evidence\n" + "0.6 = Contains useful specific information but requires additional context\n" + "1.0 = Contains critical evidence or specific details that directly answer the question\n\n" + f"Question: {question}\n\n" + f"Text to evaluate:\n{chunk_text}\n\n" + "Output only the numeric score (0.0-1.0):" + ) ) score = float(response.message.content.strip()) score = max(0.0, min(1.0, score)) - log_analysis_step(f"Computed relevance score: {score:.4f}") + logger.debug(f"[ANALYSIS] Computed relevance score: {score:.4f}") return score - except Exception as e: - log_analysis_step(f"Error scoring chunk relevance: {str(e)}", "error") + except Exception as e: # noqa: BLE001 + logger.error(f"[ANALYSIS] Error scoring chunk relevance: {e!s}") return 0.0 async def score_chunk_relevance_batch(self, question: str, chunks: List[Dict], single_call: bool = True) -> List[float]: @@ -432,49 +411,44 @@ async def score_chunk_relevance_batch(self, question: str, chunks: List[Dict], s try: if single_call: # Batch scoring - all chunks in one call - chunks_text = "\n\n".join([f"[CHUNK {i+1}]\n{chunk['text']}" for i, chunk in enumerate(chunks)]) + chunks_text = "\n\n".join([f"[CHUNK {i + 1}]\n{chunk['text']}" for i, chunk in enumerate(chunks)]) response = await self.llm.achat( - prompt=f"""As a senior equity analyst with expertise in climate science evaluating a company's sustainability report, you are tasked with evaluating text fragments for their usefulness in answering specific TCFD questions. - -Your task is to score the relevance and quality of evidence in each text fragment marked as [CHUNK X]. Consider: - -1. Specificity and Concreteness: - - Quantitative data and specific metrics (highest value) - - Concrete policies and procedures - - Specific commitments with timelines - - General statements or vague claims (lowest value) - -2. Evidence Quality: - - Verifiable data and third-party verification - - Clear methodologies and frameworks - - Specific examples and case studies - - Unsubstantiated claims (lowest value) - -3. Direct Relevance: - - Direct answers to the question components - - Related but indirect information - - Contextual background - - Unrelated information (lowest value) - -4. Disclosure Quality: - - Comprehensive and transparent disclosure - - Balanced reporting (both positive and negative) - - Clear acknowledgment of limitations - - Potential greenwashing or selective disclosure (lowest value) - -For each chunk marked [CHUNK X], provide a score from 0.0 to 1.0 where: -0.0 = Not useful (generic statements, unrelated content) -0.3 = Contains relevant context but no specific evidence -0.6 = Contains useful specific information but requires additional context -1.0 = Contains critical evidence or specific details that directly answer the question - -Question: {question} - -Text fragments to evaluate: -{chunks_text} - -Output only the scores, one per line, in order:""" + prompt=( + "As a senior equity analyst with expertise in climate science evaluating a company's " + "sustainability report, you are tasked with evaluating text fragments for their " + "usefulness in answering specific TCFD questions.\n\n" + "Your task is to score the relevance and quality of evidence in each text fragment " + "marked as [CHUNK X]. Consider:\n\n" + "1. Specificity and Concreteness:\n" + " - Quantitative data and specific metrics (highest value)\n" + " - Concrete policies and procedures\n" + " - Specific commitments with timelines\n" + " - General statements or vague claims (lowest value)\n\n" + "2. Evidence Quality:\n" + " - Verifiable data and third-party verification\n" + " - Clear methodologies and frameworks\n" + " - Specific examples and case studies\n" + " - Unsubstantiated claims (lowest value)\n\n" + "3. Direct Relevance:\n" + " - Direct answers to the question components\n" + " - Related but indirect information\n" + " - Contextual background\n" + " - Unrelated information (lowest value)\n\n" + "4. Disclosure Quality:\n" + " - Comprehensive and transparent disclosure\n" + " - Balanced reporting (both positive and negative)\n" + " - Clear acknowledgment of limitations\n" + " - Potential greenwashing or selective disclosure (lowest value)\n\n" + "For each chunk marked [CHUNK X], provide a score from 0.0 to 1.0 where:\n" + "0.0 = Not useful (generic statements, unrelated content)\n" + "0.3 = Contains relevant context but no specific evidence\n" + "0.6 = Contains useful specific information but requires additional context\n" + "1.0 = Contains critical evidence or specific details that directly answer the question\n\n" + f"Question: {question}\n\n" + f"Text fragments to evaluate:\n{chunks_text}\n\n" + "Output only the scores, one per line, in order:" + ) ) # Parse scores from response @@ -483,8 +457,8 @@ async def score_chunk_relevance_batch(self, question: str, chunks: List[Dict], s if len(scores) != len(chunks): raise ValueError(f"Got {len(scores)} scores for {len(chunks)} chunks") return scores - except Exception as e: - log_analysis_step(f"Error parsing batch scores: {str(e)}", "error") + except Exception as e: # noqa: BLE001 + logger.error(f"[ANALYSIS] Error parsing batch scores: {e!s}") return [0.0] * len(chunks) else: @@ -493,18 +467,18 @@ async def score_chunk_relevance_batch(self, question: str, chunks: List[Dict], s for i, chunk in enumerate(chunks): score = await self.score_chunk_relevance(question, chunk["text"]) scores.append(score) - log_analysis_step(f"Scored chunk {i+1}/{len(chunks)}: {score:.2f}") + logger.debug(f"[ANALYSIS] Scored chunk {i + 1}/{len(chunks)}: {score:.2f}") return scores - except Exception as e: - log_analysis_step(f"Error in batch scoring: {str(e)}", "error") + except Exception as e: # noqa: BLE001 + logger.error(f"[ANALYSIS] Error in batch scoring: {e!s}") return [0.0] * len(chunks) def _load_cached_answers(self, file_path: str) -> Dict: """Load cached answers for a file with exact configuration match""" try: # Log current configuration - logger.info(f"Current configuration:") + logger.info("Current configuration:") logger.info(f"- Chunk size: {self.chunk_params['chunk_size']}") logger.info(f"- Overlap: {self.chunk_params['chunk_overlap']}") logger.info(f"- Top K: {self.chunk_params['top_k']}") @@ -521,24 +495,31 @@ def _load_cached_answers(self, file_path: str) -> Dict: logger.info(f"- {cf.name}") # Generate cache key for current configuration - cache_key = f"cs{self.chunk_params['chunk_size']}_ov{self.chunk_params['chunk_overlap']}_tk{self.chunk_params['top_k']}_m{model_name}_qs{self.question_set}" + cache_key = ( + f"cs{self.chunk_params['chunk_size']}" + f"_ov{self.chunk_params['chunk_overlap']}" + f"_tk{self.chunk_params['top_k']}" + f"_m{model_name}" + f"_qs{self.question_set}" + ) file_stem = Path(file_path).stem cache_file = Path(self.cache_path) / f"{file_stem}_{cache_key}.json" logger.info(f"Looking for cache file: {cache_file}") if not cache_file.exists(): - logger.info(f"No cache file found for current configuration") + logger.info("No cache file found for current configuration") return {} with open(cache_file, "r") as f: cached_data = json.load(f) logger.info(f"Loaded cache data with keys: {list(cached_data.keys())}") - logger.info(f"Cache data structure: {json.dumps(cached_data, indent=2)[:500]}...") # Show first 500 chars + cache_preview = json.dumps(cached_data, indent=2)[:500] + logger.info("Cache data structure: %s...", cache_preview) # Show first 500 chars return cached_data - except Exception as e: - logger.error(f"Error loading cache: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error loading cache: {e!s}") return {} def _validate_cache_filename(self, filename: str) -> bool: @@ -560,7 +541,7 @@ def _save_cached_answers(self, file_path: str, answers: Dict) -> None: self._answers_cache[cache_key] = answers logger.info(f"[ANALYSIS] ✓ Cache SAVE: Saved answers to {cache_path}") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning(f"[ANALYSIS] Cache ERROR: Failed to save answers: {e}") async def process_document( @@ -587,10 +568,16 @@ async def process_document( logger.info(f"[ANALYSIS] Starting document processing for {file_path}") logger.info(f"[ANALYSIS] Selected questions: {selected_questions}") logger.info( - f"[ANALYSIS] LLM scoring: {use_llm_scoring}, Single call: {single_call}, Force recompute: {force_recompute}" + "[ANALYSIS] LLM scoring: %s, Single call: %s, Force recompute: %s", + use_llm_scoring, + single_call, + force_recompute, ) logger.info( - f"[ANALYSIS] Current chunk parameters: size={self.chunk_params['chunk_size']}, overlap={self.chunk_params['chunk_overlap']}, top_k={self.chunk_params['top_k']}" + "[ANALYSIS] Current chunk parameters: size=%s, overlap=%s, top_k=%s", + self.chunk_params["chunk_size"], + self.chunk_params["chunk_overlap"], + self.chunk_params["top_k"], ) if not self.use_backend_llm and self.llm is None: @@ -631,7 +618,10 @@ async def process_document( logger.info(f"[ANALYSIS] Saved {len(chunks)} pre-retrieved chunks to cache") else: logger.info( - f"[ANALYSIS] Getting document chunks from cache for {file_path} with size={self.chunk_params['chunk_size']}, overlap={self.chunk_params['chunk_overlap']}" + "[ANALYSIS] Getting document chunks from cache for %s with size=%s, overlap=%s", + file_path, + self.chunk_params["chunk_size"], + self.chunk_params["chunk_overlap"], ) chunks = self.cache_manager.get_document_chunks( file_path=file_path, @@ -641,15 +631,19 @@ async def process_document( logger.info(f"[ANALYSIS] Retrieved {len(chunks)} chunks from cache") if not chunks: - logger.info(f"[ANALYSIS] No chunks found in cache with current parameters, creating new chunks") + logger.info("[ANALYSIS] No chunks found in cache with current parameters, creating new chunks") # If no chunks in cache with current parameters, create them # Check if file_path is a URN (backend resource) if file_path.startswith("urn:report-analyst:backend:"): logger.warning( - f"[ANALYSIS] URN detected but no pre-retrieved chunks provided. Cannot process backend resource without chunks." + "[ANALYSIS] URN detected but no pre-retrieved chunks provided. " + "Cannot process backend resource without chunks." ) yield { - "error": "Backend resource requires pre-retrieved chunks. Please ensure chunks are retrieved from backend first." + "error": ( + "Backend resource requires pre-retrieved chunks. " + "Please ensure chunks are retrieved from backend first." + ) } return chunks = self._create_chunks(file_path) @@ -676,7 +670,22 @@ async def process_document( yield {"error": f"Question {question_number} not found"} continue - question_id = f"{self.question_set}_{question_number}" + # Use the same prefix extraction logic as get_question_by_number + question_set_mapping = { + "everest": "ev", + "tcfd": "tcfd", + "s4m": "s4m", + "lucia": "lucia", + "climretrieve": "climretr", + } + question_prefix = question_set_mapping.get(self.question_set, self.question_set) + # If still not found in mapping, try to extract prefix from actual question IDs + if question_prefix == self.question_set and self.questions: + first_qid = next(iter(self.questions.keys()), "") + if first_qid and "_" in first_qid: + question_prefix = first_qid.split("_")[0] + + question_id = f"{question_prefix}_{question_number}" logger.info(f"[ANALYSIS] Question ID: {question_id}") yield {"status": f"Processing question {question_number}: {question_data['text'][:50]}..."} @@ -704,16 +713,16 @@ async def process_document( for i, chunk in enumerate(similar_chunks): if i < len(llm_scores): chunk["llm_score"] = llm_scores[i] - logger.debug(f"Applied LLM score {llm_scores[i]:.3f} to chunk {i+1}") + logger.debug(f"Applied LLM score {llm_scores[i]:.3f} to chunk {i + 1}") else: chunk["llm_score"] = 0.0 - logger.warning(f"No LLM score available for chunk {i+1}") + logger.warning(f"No LLM score available for chunk {i + 1}") logger.info(f"[ANALYSIS] Applied LLM scores to {len(similar_chunks)} chunks") except Exception as e: logger.error( - f"[ANALYSIS] Error applying LLM scores: {str(e)}", + f"[ANALYSIS] Error applying LLM scores: {e!s}", exc_info=True, ) # Set default scores if LLM scoring fails @@ -761,8 +770,12 @@ async def process_document( "metadata": similar_chunks[chunk_idx]["metadata"], } ) + evidence_preview = evidence.get("text", "")[:100] logger.info( - f"Added evidence {evidence_idx + 1} from chunk {chunk_num}: {evidence.get('text', '')[:100]}..." + "Added evidence %s from chunk %s: %s...", + evidence_idx + 1, + chunk_num, + evidence_preview, ) # Replace evidence array with processed items @@ -804,14 +817,14 @@ async def process_document( except Exception as e: logger.error( - f"[ANALYSIS] Error processing question {question_number}: {str(e)}", + f"[ANALYSIS] Error processing question {question_number}: {e!s}", exc_info=True, ) - yield {"error": f"Error processing question {question_number}: {str(e)}"} + yield {"error": f"Error processing question {question_number}: {e!s}"} except Exception as e: - logger.error(f"[ANALYSIS] Error processing document: {str(e)}", exc_info=True) - yield {"error": f"Error processing document: {str(e)}"} + logger.error(f"[ANALYSIS] Error processing document: {e!s}", exc_info=True) + yield {"error": f"Error processing document: {e!s}"} def _create_chunks(self, file_path: str) -> List[Dict[str, Any]]: """Create document chunks with embeddings""" @@ -847,7 +860,7 @@ def _create_chunks(self, file_path: str) -> List[Dict[str, Any]]: for i in range(0, len(text_chunks), BATCH_SIZE): batch = text_chunks[i : i + BATCH_SIZE] - logger.info(f"Computing embeddings for batch {i//BATCH_SIZE + 1}/{(len(text_chunks)-1)//BATCH_SIZE + 1}") + logger.info(f"Computing embeddings for batch {i // BATCH_SIZE + 1}/{(len(text_chunks) - 1) // BATCH_SIZE + 1}") # Get text from batch and clean it batch_texts = [] @@ -859,7 +872,7 @@ def _create_chunks(self, file_path: str) -> List[Dict[str, Any]]: text = " ".join(text.replace("\x00", "").split()) batch_texts.append(text) else: - logger.warning(f"Skipping empty or invalid chunk") + logger.warning("Skipping empty or invalid chunk") continue try: @@ -870,7 +883,7 @@ def _create_chunks(self, file_path: str) -> List[Dict[str, Any]]: logger.info(f"Successfully computed {len(batch_embeddings)} embeddings") # Create chunk dictionaries with embeddings - for chunk, embedding in zip(batch, batch_embeddings): + for chunk, embedding in zip(batch, batch_embeddings, strict=False): if embedding is not None: # Only add chunks with valid embeddings chunk_dict = { "text": chunk.text, @@ -882,10 +895,10 @@ def _create_chunks(self, file_path: str) -> List[Dict[str, Any]]: chunks_data.append(chunk_dict) logger.debug(f"Added chunk with text length {len(chunk.text)}") else: - logger.warning(f"Skipping chunk - embedding is None") + logger.warning("Skipping chunk - embedding is None") except Exception as e: - logger.error(f"Error computing embeddings for batch: {str(e)}", exc_info=True) + logger.error(f"Error computing embeddings for batch: {e!s}", exc_info=True) # Continue with next batch, storing chunks without embeddings for chunk in batch: chunk_dict = { @@ -896,7 +909,7 @@ def _create_chunks(self, file_path: str) -> List[Dict[str, Any]]: "computed_score": 0.0, } chunks_data.append(chunk_dict) - logger.warning(f"Added chunk without embedding due to error") + logger.warning("Added chunk without embedding due to error") # Log embedding statistics chunks_with_embeddings = sum(1 for c in chunks_data if c["embedding"] is not None) @@ -908,16 +921,16 @@ def _create_chunks(self, file_path: str) -> List[Dict[str, Any]]: logger.info(f"Saving {len(valid_chunks)} valid chunks to cache") try: self.cache_manager.save_vectors(file_path, valid_chunks) - logger.info(f"Successfully saved chunks and vectors to cache") + logger.info("Successfully saved chunks and vectors to cache") except Exception as e: - logger.error(f"Failed to save vectors to cache: {str(e)}", exc_info=True) + logger.error(f"Failed to save vectors to cache: {e!s}", exc_info=True) else: logger.warning("No valid chunks to save to cache") return chunks_data except Exception as e: - logger.error(f"Error creating chunks: {str(e)}", exc_info=True) + logger.error(f"Error creating chunks: {e!s}", exc_info=True) raise async def _analyze_chunks( @@ -933,7 +946,7 @@ async def _analyze_chunks( # Process chunks first processed_chunks = [] for i, chunk in enumerate(chunks): - logger.debug(f"Processing chunk {i+1}/{len(chunks)}") + logger.debug(f"Processing chunk {i + 1}/{len(chunks)}") # Get similarity score from either 'score' (from vector store) or 'similarity_score' (from cache) similarity_score = chunk.get("score", chunk.get("similarity_score", 0.0)) @@ -987,7 +1000,7 @@ async def _analyze_chunks( logger.info(response_text) logger.info("=== End LLM Response ===") except Exception as e: - logger.error(f"Error getting LLM response: {str(e)}") + logger.error(f"Error getting LLM response: {e!s}") raise # Parse the response @@ -1058,8 +1071,12 @@ async def _analyze_chunks( "metadata": processed_chunks[chunk_idx]["metadata"], } ) + evidence_preview = evidence.get("text", "")[:100] logger.info( - f"Added evidence {evidence_idx + 1} from chunk {chunk_num}: {evidence.get('text', '')[:100]}..." + "Added evidence %s from chunk %s: %s...", + evidence_idx + 1, + chunk_num, + evidence_preview, ) # Replace evidence array with processed items @@ -1074,9 +1091,9 @@ async def _analyze_chunks( return result except Exception as e: - logger.error(f"Error analyzing chunks: {str(e)}", exc_info=True) + logger.error(f"Error analyzing chunks: {e!s}", exc_info=True) return { - "ANSWER": f"Error analyzing document: {str(e)}", + "ANSWER": f"Error analyzing document: {e!s}", "SCORE": 0, "EVIDENCE": [], "GAPS": ["Error during analysis"], @@ -1095,21 +1112,15 @@ def _load_questions(self) -> dict: Path.cwd() / "questionsets" / f"{self.question_set}_questions.yaml", # current working directory ] - log_analysis_step(f"Looking for {self.question_set}_questions.yaml in:") - for path in possible_paths: - log_analysis_step(f"- {path.resolve()}") - yaml_file = None for path in possible_paths: if path.exists(): yaml_file = path - log_analysis_step(f"✓ Found questions file at: {path.resolve()}") break if not yaml_file: - log_analysis_step( - f"Could not find questions file for {self.question_set} in any of: {[str(p) for p in possible_paths]}", - "error", + logger.error( + f"Could not find questions file for {self.question_set} in any of: {[str(p) for p in possible_paths]}" ) return {} @@ -1129,11 +1140,11 @@ def _load_questions(self) -> dict: } log_analysis_step(f"Added question {q_id}: {questions[q_id]['text'][:50]}...") - log_analysis_step(f"✓ Loaded {len(questions)} questions for {self.question_set}") + log_analysis_step(f"Loaded {len(questions)} questions for {self.question_set}") log_analysis_step(f"Available question IDs: {list(questions.keys())}") return questions except Exception as e: - log_analysis_step(f"Error loading questions: {str(e)}", "error") + logger.error(f"Error loading questions: {e!s}") logger.exception("Full error:") # This will log the full traceback return {} @@ -1146,18 +1157,31 @@ def get_question_by_number(self, number: int) -> Optional[Dict]: "tcfd": "tcfd", "s4m": "s4m", "lucia": "lucia", + "climretrieve": "climretr", # Map climretrieve to climretr shortcut } # Get the correct prefix for the question set question_prefix = question_set_mapping.get(self.question_set, self.question_set) - question_key = f"{question_prefix}_{number}" + # If still not found in mapping, try to extract prefix from actual question IDs + if question_prefix == self.question_set and self.questions: + # Extract prefix from first question ID (e.g., "climretr_1" -> "climretr") + first_qid = next(iter(self.questions.keys()), "") + if first_qid and "_" in first_qid: + extracted_prefix = first_qid.split("_")[0] + logger.info( + f"[ANALYSIS] Extracted prefix '{extracted_prefix}' from question IDs " + f"(question_set='{self.question_set}')" + ) + question_prefix = extracted_prefix + + question_key = f"{question_prefix}_{number}" logger.debug(f"Looking for question {number} with key: {question_key}") logger.debug(f"Available question keys: {list(self.questions.keys())}") return self.questions.get(question_key) except Exception as e: - log_analysis_step(f"Error getting question {number}: {str(e)}", "error") + logger.error(f"Error getting question {number}: {e!s}") logger.exception("Full error:") return None @@ -1174,11 +1198,11 @@ def update_parameters(self, chunk_size: int, chunk_overlap: int, top_k: int): # Recreate text splitter with new parameters self.text_splitter = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap) - logger.info(f"Updated parameters and recreated text splitter") + logger.info("Updated parameters and recreated text splitter") def update_llm_model(self, model_name: str): """Update the LLM model.""" - log_analysis_step(f"Updating LLM model to: {model_name}") + logger.info(f"Updating LLM model to: {model_name}") self.default_model = model_name if not self._has_key_for_model(model_name): @@ -1243,8 +1267,8 @@ def check_step_completion(self, file_path: str) -> Dict[str, bool]: "analysis": step4_complete, } - except Exception as e: - logger.error(f"Error checking step completion: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error checking step completion: {e!s}") return { "chunks": False, "embeddings": False, @@ -1287,7 +1311,7 @@ def _parse_config_from_filename(self, filename: str) -> Dict[str, Any]: return config - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning(f"Error parsing config from filename {filename}: {e}") return config @@ -1325,7 +1349,7 @@ async def _get_similar_chunks(self, query_text: str, chunks: List[Dict], top_k: return similar_chunks except Exception as e: - logger.error(f"Error getting similar chunks: {str(e)}", exc_info=True) + logger.error(f"Error getting similar chunks: {e!s}", exc_info=True) return [] def _parse_analysis_response(self, response_text: str) -> Dict[str, Any]: @@ -1466,9 +1490,9 @@ def _parse_analysis_response(self, response_text: str) -> Dict[str, Any]: return result except Exception as e: - logger.error(f"Error parsing analysis response: {str(e)}", exc_info=True) + logger.error(f"Error parsing analysis response: {e!s}", exc_info=True) return { - "ANSWER": f"Error parsing analysis: {str(e)}", + "ANSWER": f"Error parsing analysis: {e!s}", "SCORE": 0, "EVIDENCE": [], "GAPS": ["Error during analysis"], diff --git a/report_analyst/core/benchmark/__init__.py b/report_analyst/core/benchmark/__init__.py new file mode 100644 index 00000000..ef3313f0 --- /dev/null +++ b/report_analyst/core/benchmark/__init__.py @@ -0,0 +1 @@ +# Benchmark package for evaluation and annotation functionality diff --git a/report_analyst/core/benchmark/classification_calibration.py b/report_analyst/core/benchmark/classification_calibration.py new file mode 100644 index 00000000..c9120f32 --- /dev/null +++ b/report_analyst/core/benchmark/classification_calibration.py @@ -0,0 +1,233 @@ +import logging +from typing import Dict, Iterable, List + +import numpy as np +import pandas as pd +from sklearn.metrics import ( + brier_score_loss, + classification_report, + f1_score, + roc_auc_score, +) + +logger = logging.getLogger(__name__) + + +def minmax_normalize(series: pd.Series) -> np.ndarray: + """Min-max normalize a score series to [0, 1]. + + If all values are identical, returns an array of 0.5 (uninformative probability). + """ + arr = pd.to_numeric(series, errors="coerce").to_numpy(dtype=float) + if arr.size == 0: + return arr + min_val = np.nanmin(arr) + max_val = np.nanmax(arr) + if not np.isfinite(min_val) or not np.isfinite(max_val): + return np.zeros_like(arr, dtype=float) + if max_val == min_val: + return np.full_like(arr, 0.5, dtype=float) + return (arr - min_val) / (max_val - min_val) + + +def expected_calibration_error(y_true: np.ndarray, y_prob: np.ndarray, n_bins: int = 100) -> float: + """Compute Expected Calibration Error (ECE) for binary labels and probabilities.""" + if y_true.size == 0: + return 0.0 + + y_true = np.asarray(y_true, dtype=float) + y_prob = np.asarray(y_prob, dtype=float) + + bins = np.linspace(0.0, 1.0, n_bins + 1) + bin_ids = np.digitize(y_prob, bins) - 1 + + ece = 0.0 + n = len(y_true) + + for b in range(n_bins): + mask = bin_ids == b + if np.any(mask): + acc = y_true[mask].mean() + conf = y_prob[mask].mean() + ece += (mask.sum() / n) * abs(acc - conf) + + return float(ece) + + +def _compute_metrics_for_pair( + df: pd.DataFrame, + ground_truth_col: str, + score_col: str, + n_bins: int = 100, +) -> Dict[str, float]: + """Compute calibration and F1 metrics for a single (ground_truth, score) pair.""" + if ground_truth_col not in df.columns or score_col not in df.columns: + raise ValueError(f"Columns '{ground_truth_col}' and/or '{score_col}' not found in DataFrame") + + mask = df[ground_truth_col].notna() & df[score_col].notna() + if not mask.any(): + logger.warning( + "No overlapping non-NaN rows for columns '%s' and '%s'", + ground_truth_col, + score_col, + ) + return { + "ground_truth": ground_truth_col, + "prediction": score_col, + "ece": np.nan, + "ece2": np.nan, + "brier": np.nan, + "brier2": np.nan, + "auroc": np.nan, + "f1": np.nan, + "f1_1": np.nan, + "f2_2": np.nan, + } + + y_true = pd.to_numeric(df.loc[mask, ground_truth_col], errors="coerce").astype(int) + y_score_raw = pd.to_numeric(df.loc[mask, score_col], errors="coerce") + + # Binary ground truth versions + y_true_bin1 = (y_true > 0).astype(int).to_numpy() + y_true_bin2 = (y_true > 1).astype(int).to_numpy() + + # Scores and thresholds + y_score_minmax = minmax_normalize(y_score_raw) + y_score_ordinal = np.where( + y_score_raw > 1, + 2, + np.where(y_score_raw > 0, 1, 0), + ).astype(int) + y_score_bin1 = (y_score_raw > 0).astype(int).to_numpy() + y_score_bin2 = (y_score_raw > 1).astype(int).to_numpy() + + # Calibration scores (handle edge cases defensively) + ece = expected_calibration_error(y_true_bin1, y_score_minmax, n_bins=n_bins) + ece2 = expected_calibration_error(y_true_bin2, y_score_minmax, n_bins=n_bins) + try: + brier = brier_score_loss(y_true_bin1, y_score_minmax) + except ValueError: + brier = np.nan + try: + brier2 = brier_score_loss(y_true_bin2, y_score_minmax) + except ValueError: + brier2 = np.nan + + # AUC can fail if only one class is present + try: + auroc = roc_auc_score(y_true_bin1, y_score_minmax) + except ValueError: + auroc = np.nan + + # F1 scores + f1_macro = f1_score(y_true, y_score_ordinal, average="macro") + f1_1 = f1_score(y_true_bin1, y_score_bin1) + f1_2 = f1_score(y_true_bin2, y_score_bin2) + + return { + "ground_truth": ground_truth_col, + "prediction": score_col, + "ece": float(ece), + "ece2": float(ece2), + "brier": float(brier), + "brier2": float(brier2), + "auroc": float(auroc), + "f1": float(f1_macro), + "f1_1": float(f1_1), + "f2_2": float(f1_2), + } + + +def compute_calibration_metrics( + df: pd.DataFrame, + ground_truth_col: str, + score_cols: Iterable[str], + n_bins: int = 100, +) -> pd.DataFrame: + """Compute calibration metrics for multiple score columns. + + Returns a DataFrame with one row per (ground_truth, prediction) pair. + """ + rows: List[Dict[str, float]] = [] + for col in score_cols: + try: + row = _compute_metrics_for_pair( + df=df, + ground_truth_col=ground_truth_col, + score_col=col, + n_bins=n_bins, + ) + rows.append(row) + except Exception as exc: # pragma: no cover - defensive + logger.exception( + "Failed to compute calibration metrics for ground_truth=%s, prediction=%s: %s", + ground_truth_col, + col, + exc, + ) + rows.append( + { + "ground_truth": ground_truth_col, + "prediction": col, + "ece": np.nan, + "ece2": np.nan, + "brier": np.nan, + "brier2": np.nan, + "auroc": np.nan, + "f1": np.nan, + "f1_1": np.nan, + "f2_2": np.nan, + } + ) + + return pd.DataFrame(rows) + + +def compute_classification_report( + df: pd.DataFrame, + ground_truth_col: str, + score_col: str, +) -> Dict: + """Compute a 3-class classification report from ground-truth labels and scores. + + Uses the same 2/1/0 thresholding on the score column as in the notebook. + """ + if ground_truth_col not in df.columns or score_col not in df.columns: + raise ValueError(f"Columns '{ground_truth_col}' and/or '{score_col}' not found in DataFrame") + + mask = df[ground_truth_col].notna() & df[score_col].notna() + if not mask.any(): + return {} + + y_true = pd.to_numeric(df.loc[mask, ground_truth_col], errors="coerce").astype(int).to_numpy() + y_score = pd.to_numeric(df.loc[mask, score_col], errors="coerce") + y_pred = np.where( + y_score > 1, + 2, + np.where(y_score > 0, 1, 0), + ).astype(int) + + # Choose human-friendly target names + if ground_truth_col == "usefulness": + target_names = ["not useful (0)", "partially useful (1)", "useful (2)"] + else: + target_names = ["irrelevant (0)", "partially relevant (1)", "relevant (2)"] + + report = classification_report( + y_true, + y_pred, + labels=[0, 1, 2], + target_names=target_names, + digits=3, + output_dict=True, + zero_division=0, + ) + return report + + +__all__ = [ + "compute_calibration_metrics", + "compute_classification_report", + "expected_calibration_error", + "minmax_normalize", +] diff --git a/report_analyst/core/benchmark/dataset_loader.py b/report_analyst/core/benchmark/dataset_loader.py new file mode 100644 index 00000000..dd672e40 --- /dev/null +++ b/report_analyst/core/benchmark/dataset_loader.py @@ -0,0 +1,184 @@ +import hashlib +import json +import logging +from pathlib import Path +from typing import Dict, List + +import yaml + +from ...models.benchmark import ( + BenchmarkDatasetContent, + BenchmarkQuestion, + GroundTruthChunk, +) + +logger = logging.getLogger(__name__) + + +class DatasetValidationError(Exception): + """Raised when dataset validation fails""" + + pass + + +class DatasetLoader: + """Loads and validates benchmark datasets""" + + def __init__(self): + self.supported_formats = [".yaml", ".yml", ".json"] + + def load_dataset(self, file_path: str) -> BenchmarkDatasetContent: + """Load and validate a benchmark dataset from file""" + path = Path(file_path) + + if not path.exists(): + raise FileNotFoundError(f"Dataset file not found: {file_path}") + + if path.suffix.lower() not in self.supported_formats: + raise ValueError(f"Unsupported file format: {path.suffix}. Supported: {self.supported_formats}") + + # Load raw data + raw_data = self._load_raw_data(path) + + # Validate and parse + dataset = self._validate_and_parse(raw_data) + + logger.info(f"Successfully loaded dataset '{dataset.name}' with {len(dataset.questions)} questions") + return dataset + + def _load_raw_data(self, path: Path) -> Dict: + """Load raw data from file""" + try: + with open(path, "r", encoding="utf-8") as f: + if path.suffix.lower() in [".yaml", ".yml"]: + return yaml.safe_load(f) + else: + return json.load(f) + except Exception as e: + raise DatasetValidationError(f"Failed to load file {path}: {e}") from e + + def _validate_and_parse(self, raw_data: Dict) -> BenchmarkDatasetContent: + """Validate raw data and parse into structured format""" + try: + # Validate required top-level fields + required_fields = [ + "dataset_id", + "name", + "description", + "version", + "question_set", + "questions", + ] + for field in required_fields: + if field not in raw_data: + raise DatasetValidationError(f"Missing required field: {field}") + + # Parse questions + questions = [] + for q_data in raw_data["questions"]: + question = self._parse_question(q_data) + questions.append(question) + + return BenchmarkDatasetContent( + dataset_id=raw_data["dataset_id"], + name=raw_data["name"], + description=raw_data["description"], + version=raw_data["version"], + question_set=raw_data["question_set"], + created_at=raw_data.get("created_at", ""), + questions=questions, + ) + + except Exception as e: + raise DatasetValidationError(f"Dataset validation failed: {e}") from e + + def _parse_question(self, q_data: Dict) -> BenchmarkQuestion: + """Parse a single question from raw data""" + if "question_id" not in q_data: + raise DatasetValidationError("Question missing 'question_id'") + if "question_text" not in q_data: + raise DatasetValidationError("Question missing 'question_text'") + if "ground_truth_chunks" not in q_data: + raise DatasetValidationError("Question missing 'ground_truth_chunks'") + + # Parse ground truth chunks + chunks = [] + for chunk_data in q_data["ground_truth_chunks"]: + chunk = self._parse_ground_truth_chunk(chunk_data) + chunks.append(chunk) + + return BenchmarkQuestion( + question_id=q_data["question_id"], + question_text=q_data["question_text"], + ground_truth_chunks=chunks, + ) + + def _parse_ground_truth_chunk(self, chunk_data: Dict) -> GroundTruthChunk: + """Parse a ground truth chunk from raw data""" + required_fields = ["chunk_id", "relevance_score", "is_evidence"] + for field in required_fields: + if field not in chunk_data: + raise DatasetValidationError(f"Ground truth chunk missing '{field}'") + + # Validate relevance score + score = chunk_data["relevance_score"] + if not isinstance(score, (int, float)) or score < 0.0 or score > 1.0: + raise DatasetValidationError(f"Invalid relevance_score: {score}. Must be between 0.0 and 1.0") + + return GroundTruthChunk( + chunk_id=chunk_data["chunk_id"], + relevance_score=float(score), + is_evidence=bool(chunk_data["is_evidence"]), + evidence_order=chunk_data.get("evidence_order"), + annotation_notes=chunk_data.get("annotation_notes"), + ) + + def validate_dataset_consistency(self, dataset: BenchmarkDatasetContent) -> List[str]: + """Validate dataset consistency and return list of warnings""" + warnings = [] + + # Check for duplicate question IDs + question_ids = [q.question_id for q in dataset.questions] + if len(question_ids) != len(set(question_ids)): + warnings.append("Duplicate question IDs found") + + # Check evidence ordering + for question in dataset.questions: + evidence_chunks = [c for c in question.ground_truth_chunks if c.is_evidence] + if evidence_chunks: + orders = [c.evidence_order for c in evidence_chunks if c.evidence_order is not None] + if len(orders) != len(set(orders)): + warnings.append(f"Duplicate evidence orders in question {question.question_id}") + + # Check for questions without any relevant chunks + for question in dataset.questions: + relevant_chunks = [c for c in question.ground_truth_chunks if c.relevance_score > 0.0] + if not relevant_chunks: + warnings.append(f"Question {question.question_id} has no relevant chunks") + + return warnings + + def generate_dataset_hash(self, dataset: BenchmarkDatasetContent) -> str: + """Generate a hash for the dataset content""" + # Create a deterministic string representation + content = { + "dataset_id": dataset.dataset_id, + "version": dataset.version, + "questions": [], + } + + for question in sorted(dataset.questions, key=lambda q: q.question_id): + q_data = {"question_id": question.question_id, "chunks": []} + for chunk in sorted(question.ground_truth_chunks, key=lambda c: c.chunk_id): + q_data["chunks"].append( + { + "chunk_id": chunk.chunk_id, + "relevance_score": chunk.relevance_score, + "is_evidence": chunk.is_evidence, + "evidence_order": chunk.evidence_order, + } + ) + content["questions"].append(q_data) + + content_str = json.dumps(content, sort_keys=True) + return hashlib.sha256(content_str.encode()).hexdigest()[:16] diff --git a/report_analyst/core/benchmark/dataset_mapper.py b/report_analyst/core/benchmark/dataset_mapper.py new file mode 100644 index 00000000..653ddceb --- /dev/null +++ b/report_analyst/core/benchmark/dataset_mapper.py @@ -0,0 +1,462 @@ +import hashlib +import logging +from importlib import import_module +from pathlib import Path +from typing import Any, Dict, List, Optional, Type + +import pandas as pd +import yaml + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Shared ID helpers (moved from scripts/align_benchmark_datasets.py) +# --------------------------------------------------------------------------- + + +def generate_query_id(document: str, question: str, separator: str = "|||") -> str: + """ + Generate a consistent query_id from document and question. + + Args: + document: Document/report name + question: Question text + separator: Separator between document and question + + Returns: + query_id string + """ + # Normalize: strip whitespace, handle None/NaN + doc = str(document).strip() if pd.notna(document) else "" + q = str(question).strip() if pd.notna(question) else "" + return f"{doc}{separator}{q}" + + +def generate_chunk_id(text: str, prefix: str = "") -> str: + """ + Generate a consistent chunk_id from text content using MD5 hash. + + Args: + text: Text content to hash + prefix: Optional prefix for chunk_id + + Returns: + chunk_id string (16-character hex hash) + """ + if pd.isna(text) or not str(text).strip(): + # Generate a hash from the string representation for empty/NaN + text_str = str(text) if pd.notna(text) else "empty" + hash_val = hashlib.md5(text_str.encode(), usedforsecurity=False).hexdigest()[:16] + return f"{prefix}{hash_val}" if prefix else hash_val + + # Normalize text: strip whitespace, lowercase for consistency + normalized = str(text).strip() + hash_val = hashlib.md5(normalized.encode(), usedforsecurity=False).hexdigest()[:16] + return f"{prefix}{hash_val}" if prefix else hash_val + + +# --------------------------------------------------------------------------- +# Core transformation helpers (adapted from align_benchmark_datasets.py) +# --------------------------------------------------------------------------- + + +def transform_ground_truth( + df: pd.DataFrame, + document_col: str = "document", + question_col: str = "question", + context_col: str = "context", + relevant_col: Optional[str] = None, + page_col: Optional[str] = None, + relevance_label_col: str = "relevance_label", +) -> pd.DataFrame: + """ + Transform ground truth dataset to consistent format. + + This is a slightly refactored version of the original helper in + scripts/align_benchmark_datasets.py, without any file I/O. It returns + an aligned DataFrame ready for evaluation. + """ + logger.info("Transforming ground truth dataset (%d rows)", len(df)) + + # Detect column names (case-insensitive) + df_cols_lower = {col.lower(): col for col in df.columns} + + # Map expected columns + doc_col = df_cols_lower.get(document_col.lower()) or df_cols_lower.get("document") + q_col = df_cols_lower.get(question_col.lower()) or df_cols_lower.get("question") + ctx_col = df_cols_lower.get(context_col.lower()) or df_cols_lower.get("context") or df_cols_lower.get("relevant") + rel_col = (df_cols_lower.get(relevant_col.lower()) if relevant_col else None) or df_cols_lower.get("relevant") + page_num_col = ( + (df_cols_lower.get(page_col.lower()) if page_col else None) + or df_cols_lower.get("page_number") + or df_cols_lower.get("page") + ) + label_col = ( + df_cols_lower.get(relevance_label_col.lower()) + or df_cols_lower.get("relevance_label") + or df_cols_lower.get("source relevance score") + or df_cols_lower.get("relevance") + or df_cols_lower.get("label") + ) + + # Validate required columns + if not doc_col: + raise ValueError(f"Could not find document column. Available: {list(df.columns)}") + if not q_col: + raise ValueError(f"Could not find question column. Available: {list(df.columns)}") + if not ctx_col and not rel_col: + raise ValueError(f"Could not find context or relevant column. Available: {list(df.columns)}") + if not label_col: + logger.warning( + "Could not find relevance label column. Will default to score=1.0. Available: %s", + list(df.columns), + ) + + # Use relevant column first (preferred), then context as fallback for chunk_id + text_col = rel_col or ctx_col + if text_col == rel_col: + logger.info("Using 'relevant' column for chunk_id generation (preferred for matching with benchmark relevant_text)") + elif text_col == ctx_col: + logger.info("Using 'context' column for chunk_id generation (fallback, 'relevant' column not found)") + + # Generate query_id + df = df.copy() + df["query_id"] = df.apply(lambda row: generate_query_id(row[doc_col], row[q_col]), axis=1) + + # Generate chunk_id from relevant text (or context if relevant not available) + df["chunk_id"] = df[text_col].apply(generate_chunk_id) + + # Generate position (1-indexed, per query) + df["position"] = df.groupby("query_id").cumcount() + 1 + + # Map relevance label to score + if label_col: + + def parse_label(val: Any) -> float: + if pd.isna(val): + return 1.0 + val_str = str(val).strip().lower() + try: + return float(val_str) + except ValueError: + if val_str in ["yes", "y", "true", "relevant", "high"]: + return 2.0 + if val_str in ["maybe", "partial", "medium"]: + return 1.0 + if val_str in ["no", "n", "false", "irrelevant", "low"]: + return 0.0 + return 1.0 + + df["score"] = df[label_col].apply(parse_label) + logger.info( + "Ground truth score distribution: %s", + dict(df["score"].value_counts().sort_index()), + ) + else: + logger.warning("No relevance label column found, defaulting all scores to 1.0") + df["score"] = 1.0 + + # Select output columns and rename preserved ones + output_cols = ["query_id", "chunk_id", "position", "score"] + preserve_cols = [doc_col, q_col] + if text_col: + preserve_cols.append(text_col) + if rel_col and rel_col != text_col and rel_col in df.columns: + preserve_cols.append(rel_col) + if ctx_col and ctx_col != text_col and ctx_col in df.columns: + preserve_cols.append(ctx_col) + if page_num_col: + preserve_cols.append(page_num_col) + if label_col and label_col in df.columns: + preserve_cols.append(label_col) + + rename_map: Dict[str, str] = { + doc_col: "document", + q_col: "question", + } + if text_col == rel_col: + rename_map[text_col] = "relevant" + elif text_col == ctx_col: + rename_map[text_col] = "context" + if rel_col and rel_col != text_col and rel_col in preserve_cols: + rename_map[rel_col] = "relevant" + if ctx_col and ctx_col != text_col and ctx_col in preserve_cols: + rename_map[ctx_col] = "context" + if page_num_col: + rename_map[page_num_col] = "page_number" + if label_col and label_col in df.columns: + rename_map[label_col] = "relevance_label" + + df_output = df[output_cols + preserve_cols].copy() + df_output = df_output.rename(columns=rename_map) + + logger.info( + "Transformed ground truth: %d rows, %d queries, %d chunks", + len(df_output), + df_output["query_id"].nunique(), + df_output["chunk_id"].nunique(), + ) + return df_output + + +def transform_benchmark_results( + df: pd.DataFrame, + report_col: str = "report", + question_col: str = "question", + paragraph_col: str = "paragraph", + relevant_text_col: str = "relevant_text", + relevance_score_col: Optional[str] = None, + label_col: Optional[str] = None, + number_col: Optional[str] = None, +) -> pd.DataFrame: + """ + Transform benchmark results dataset to consistent format. + + This is a refactored version of the original helper in + scripts/align_benchmark_datasets.py, without any file I/O. It returns + an aligned DataFrame ready for evaluation. + """ + logger.info("Transforming benchmark results dataset (%d rows)", len(df)) + + # Detect column names (case-insensitive) + df_cols_lower = {col.lower(): col for col in df.columns} + + # Map expected columns + report_col_actual = df_cols_lower.get(report_col.lower()) or df_cols_lower.get("report") or df_cols_lower.get("document") + q_col = df_cols_lower.get(question_col.lower()) or df_cols_lower.get("question") + para_col = df_cols_lower.get(paragraph_col.lower()) or df_cols_lower.get("paragraph") or df_cols_lower.get("chunk") + rel_text_col = ( + df_cols_lower.get(relevant_text_col.lower()) or df_cols_lower.get("relevant_text") or df_cols_lower.get("relevant") + ) + rel_score_col = ( + (df_cols_lower.get(relevance_score_col.lower()) if relevance_score_col else None) + or df_cols_lower.get("relevance_score") + or df_cols_lower.get("sim_text_relevance") + ) + label_col_actual = ( + (df_cols_lower.get(label_col.lower()) if label_col else None) + or df_cols_lower.get("label") + or df_cols_lower.get("relevance") + ) + num_col = ( + (df_cols_lower.get(number_col.lower()) if number_col else None) + or df_cols_lower.get("number") + or df_cols_lower.get("paragraph_number") + ) + + # Validate required columns + if not report_col_actual: + raise ValueError(f"Could not find report column. Available: {list(df.columns)}") + if not q_col: + raise ValueError(f"Could not find question column. Available: {list(df.columns)}") + if not para_col: + raise ValueError(f"Could not find paragraph column. Available: {list(df.columns)}") + + df = df.copy() + + # Generate query_id + df["query_id"] = df.apply(lambda row: generate_query_id(row[report_col_actual], row[q_col]), axis=1) + + # Generate chunk_id from paragraph (unique identifier for each retrieved paragraph) + df["chunk_id"] = df[para_col].apply(generate_chunk_id) + logger.info("Using paragraph for chunk_id generation (unique per retrieved paragraph)") + + # Generate relevant_part_id from relevant_text (for matching to ground truth relevant parts) + if rel_text_col and rel_text_col in df.columns: + df["relevant_part_id"] = df[rel_text_col].apply(generate_chunk_id) + logger.info("Using relevant_text for relevant_part_id generation (for ground truth matching)") + else: + df["relevant_part_id"] = df["chunk_id"] + logger.info("No relevant_text found, using chunk_id as relevant_part_id") + + # Generate position + if num_col and num_col in df.columns: + df["position"] = pd.to_numeric(df[num_col], errors="coerce").fillna(0).astype(int) + df["position"] = df["position"].apply(lambda x: max(1, x)) + logger.info("Using number column for position") + else: + df["position"] = df.groupby("query_id").cumcount() + 1 + logger.info("Using row order for position") + + # Optional score column (label preferred, then numeric relevance score) + if label_col_actual and label_col_actual in df.columns: + + def parse_label(val: Any) -> float: + if pd.isna(val): + return 0.0 + val_str = str(val).strip().lower() + try: + return float(val_str) + except ValueError: + if val_str in ["yes", "y", "true", "relevant", "high"]: + return 2.0 + if val_str in ["maybe", "partial", "medium"]: + return 1.0 + if val_str in ["no", "n", "false", "irrelevant", "low"]: + return 0.0 + return 0.0 + + df["score"] = df[label_col_actual].apply(parse_label) + logger.info("Using label column for score") + elif rel_score_col and rel_score_col in df.columns: + df["score"] = pd.to_numeric(df[rel_score_col], errors="coerce").fillna(0.0) + logger.info("Using numeric relevance score column for score") + else: + logger.info("No score column found - evaluation will rely on ground truth scores only") + + # Detect similarity score columns from report level dataset + relevant_text_sim_col = df_cols_lower.get("relevant_text_sim") + sim_text_relevance_col = df_cols_lower.get("sim_text_relevance") + + output_cols = ["query_id", "chunk_id", "relevant_part_id", "position"] + if "score" in df.columns: + output_cols.append("score") + + preserve_cols = [report_col_actual, q_col, para_col] + if rel_text_col and rel_text_col in df.columns: + preserve_cols.append(rel_text_col) + if label_col_actual and label_col_actual in df.columns: + preserve_cols.append(label_col_actual) + if relevant_text_sim_col and relevant_text_sim_col in df.columns: + preserve_cols.append(relevant_text_sim_col) + if sim_text_relevance_col and sim_text_relevance_col in df.columns: + preserve_cols.append(sim_text_relevance_col) + + rename_map: Dict[str, str] = { + report_col_actual: "report", + q_col: "question", + para_col: "paragraph", + } + if rel_text_col and rel_text_col in df.columns: + rename_map[rel_text_col] = "relevant_text" + if label_col_actual and label_col_actual in df.columns: + rename_map[label_col_actual] = "relevance_label" + if relevant_text_sim_col and relevant_text_sim_col in df.columns: + rename_map[relevant_text_sim_col] = "relevant_text_sim" + if sim_text_relevance_col and sim_text_relevance_col in df.columns: + rename_map[sim_text_relevance_col] = "sim_text_relevance" + + df_output = df[output_cols + preserve_cols].copy() + df_output = df_output.rename(columns=rename_map) + + logger.info( + "Transformed benchmark results: %d rows, %d queries, %d chunks", + len(df_output), + df_output["query_id"].nunique(), + df_output["chunk_id"].nunique(), + ) + return df_output + + +# --------------------------------------------------------------------------- +# Dataset mapper abstraction and factory +# --------------------------------------------------------------------------- + + +class DatasetMapper: + """Base class for dataset mappers.""" + + def __init__(self, dataset_id: str, config: Optional[Dict[str, Any]] = None): + self.dataset_id = dataset_id + self.config = config or {} + + def align_ground_truth(self, df: pd.DataFrame) -> pd.DataFrame: + raise NotImplementedError + + def align_benchmark(self, df: pd.DataFrame) -> pd.DataFrame: + raise NotImplementedError + + +class DefaultDatasetMapper(DatasetMapper): + """Default mapper that uses YAML column mappings and shared helpers.""" + + def align_ground_truth(self, df: pd.DataFrame) -> pd.DataFrame: + gt_cfg = (self.config.get("ground_truth") or {}).get("columns", {}) + + return transform_ground_truth( + df, + document_col=gt_cfg.get("document", "document"), + question_col=gt_cfg.get("question", "question"), + context_col=gt_cfg.get("context", "context"), + relevant_col=gt_cfg.get("relevant"), + page_col=gt_cfg.get("page_number"), + relevance_label_col=gt_cfg.get("relevance_label", "relevance_label"), + ) + + def align_benchmark(self, df: pd.DataFrame) -> pd.DataFrame: + bm_cfg = (self.config.get("benchmark") or {}).get("columns", {}) + + return transform_benchmark_results( + df, + report_col=bm_cfg.get("report_id", "report"), + question_col=bm_cfg.get("question", "question"), + paragraph_col=bm_cfg.get("paragraph", "paragraph"), + relevant_text_col=bm_cfg.get("relevant_text", "relevant_text"), + relevance_score_col=bm_cfg.get("relevance_score"), + label_col=bm_cfg.get("relevance_label"), + number_col=bm_cfg.get("number"), + ) + + +def _load_yaml_config(dataset_id: str) -> Dict[str, Any]: + """Load YAML config for a dataset_id from package data.""" + # Expected location: report_analyst/config/datasets/{dataset_id}.yaml + base_dir = Path(__file__).resolve().parents[2] # .../report_analyst + cfg_path = base_dir / "config" / "datasets" / f"{dataset_id}.yaml" + + if not cfg_path.exists(): + logger.warning("No dataset mapping config found for '%s' at %s", dataset_id, cfg_path) + return {"id": dataset_id} + + with cfg_path.open("r", encoding="utf-8") as f: + cfg = yaml.safe_load(f) or {} + if "id" not in cfg: + cfg["id"] = dataset_id + return cfg + + +class DatasetMapperFactory: + """Factory to construct DatasetMapper instances from YAML configs.""" + + @staticmethod + def get_mapper(dataset_id: str) -> DatasetMapper: + cfg = _load_yaml_config(dataset_id) + mapper_class_path = cfg.get("mapper_class") + + if mapper_class_path: + try: + module_path, class_name = mapper_class_path.rsplit(".", 1) + module = import_module(module_path) + cls: Type[DatasetMapper] = getattr(module, class_name) + return cls(dataset_id, cfg) + except Exception as exc: # noqa: BLE001 # pragma: no cover - defensive + logger.error( + "Failed to import custom mapper '%s' for dataset '%s': %s", + mapper_class_path, + dataset_id, + exc, + ) + + # Fallback to default mapper + return DefaultDatasetMapper(dataset_id, cfg) + + +def list_available_dataset_ids() -> List[str]: + """Return dataset IDs for which a YAML mapping config is available.""" + base_dir = Path(__file__).resolve().parents[2] / "config" / "datasets" + if not base_dir.exists(): + return [] + return sorted(p.stem for p in base_dir.glob("*.yaml")) + + +__all__ = [ + "DatasetMapper", + "DatasetMapperFactory", + "DefaultDatasetMapper", + "generate_chunk_id", + "generate_query_id", + "list_available_dataset_ids", + "transform_benchmark_results", + "transform_ground_truth", +] diff --git a/report_analyst/core/benchmark/dataset_normalizer.py b/report_analyst/core/benchmark/dataset_normalizer.py new file mode 100644 index 00000000..28dd1c1e --- /dev/null +++ b/report_analyst/core/benchmark/dataset_normalizer.py @@ -0,0 +1,188 @@ +""" +Normalize a raw DataFrame to the benchmark schema (query_id, chunk_id, position, score, paragraph) +using user-selected column mappings. Used when CSV/Excel structure does not match the expected names. +""" + +import hashlib +import logging +from typing import Any, Optional + +import pandas as pd + +from .dataset_mapper import generate_chunk_id, generate_query_id + +logger = logging.getLogger(__name__) + +# Position mode: use explicit column, infer from row order per query, or infer by sorting by score +POSITION_MODE_COLUMN = "column" +POSITION_MODE_ROW_ORDER = "row_order" +POSITION_MODE_SORT_BY_SCORE = "sort_by_score" + + +def _parse_score(val: Any) -> float: + """Coerce a value to a numeric score (for label/prediction columns).""" + if pd.isna(val): + return 0.0 + val_str = str(val).strip().lower() + try: + return float(val_str) + except ValueError: + if val_str in ["yes", "y", "true", "relevant", "high"]: + return 2.0 + if val_str in ["maybe", "partial", "medium"]: + return 1.0 + if val_str in ["no", "n", "false", "irrelevant", "low"]: + return 0.0 + return 0.0 + + +def _stable_query_id_from_text(text: str, max_len: int = 64) -> str: + """Produce a stable query_id from a single text (e.g. description) when no document column.""" + if pd.isna(text) or not str(text).strip(): + return hashlib.md5(b"empty", usedforsecurity=False).hexdigest()[:16] + s = str(text).strip() + if len(s) <= max_len: + return s + return hashlib.md5(s.encode(), usedforsecurity=False).hexdigest()[:16] + + +def make_query_id_from_columns(document: Optional[Any], question: Any, max_len: int = 64) -> str: + """ + Helper to build a stable query_id from document/question-style fields. + + - If a document is provided, we delegate to generate_query_id(document, question) + so that behaviour matches DatasetMapper / preset mappers. + - If no document is provided, we fall back to a stable ID derived from the + question/description text alone (using _stable_query_id_from_text). + """ + if document is not None and str(document).strip(): + return generate_query_id(document, question) + return _stable_query_id_from_text(question, max_len=max_len) + + +def make_chunk_id_from_text(text: Any) -> str: + """ + Helper to build a chunk_id from raw text content. + + Uses generate_chunk_id under the hood and handles empty/NaN values by + normalizing them to an empty string first. + """ + return generate_chunk_id(str(text) if pd.notna(text) else "") + + +def make_relevant_part_id_from_text(text: Any) -> str: + """ + Helper to build a relevant_part_id from a relevant span of text. + + This mirrors make_chunk_id_from_text so that relevant parts and full chunks + share the same hashing logic. When no text is provided, we hash an empty + string to get a deterministic ID. + """ + return generate_chunk_id(str(text) if pd.notna(text) else "") + + +def normalize_dataframe_for_benchmark( + df: pd.DataFrame, + query_column: str, + chunk_text_column: str, + score_column: str, + position_column: Optional[str] = None, + document_column: Optional[str] = None, + position_mode: str = POSITION_MODE_ROW_ORDER, +) -> pd.DataFrame: + """ + Build a DataFrame with standard benchmark columns from raw data and column choices. + + Produces columns: query_id, chunk_id, position, score, paragraph, question, + and optionally document. Used so that + load_flexible_dataset_from_csv(csv_content=result.to_csv()) works without schema changes. + + Args: + df: Raw DataFrame (e.g. from uploaded CSV/Excel). + query_column: Column used as query/criteria (e.g. description, question). + chunk_text_column: Column used as chunk text (e.g. chunk_text, paragraph). + score_column: Column used as label or prediction score (e.g. relevance, usefulness, model column). + position_column: Column for position/rank; used only when position_mode == "column". + document_column: Optional document/report column; if set, query_id = generate_query_id(doc, query). + position_mode: "column" | "row_order" | "sort_by_score". + - column: use position_column (must be provided). + - row_order: assign 1,2,3... by row order within each query_id. + - sort_by_score: sort by score_column descending per query, then assign position. + + Returns: + DataFrame with columns query_id, chunk_id, position, score, paragraph, question [, document]. + """ + if query_column not in df.columns: + raise ValueError(f"Query column '{query_column}' not in DataFrame columns: {list(df.columns)}") + if chunk_text_column not in df.columns: + raise ValueError(f"Chunk text column '{chunk_text_column}' not in DataFrame columns: {list(df.columns)}") + if score_column not in df.columns: + raise ValueError(f"Score column '{score_column}' not in DataFrame columns: {list(df.columns)}") + if position_mode == POSITION_MODE_COLUMN and (not position_column or position_column not in df.columns): + raise ValueError(f"Position mode is 'column' but position column '{position_column}' missing or not in DataFrame") + if document_column is not None and document_column not in df.columns: + raise ValueError(f"Document column '{document_column}' not in DataFrame columns: {list(df.columns)}") + + out = df.copy() + + # query_id + if document_column: + out["query_id"] = out.apply( + lambda row: generate_query_id( + row.get(document_column) if pd.notna(row.get(document_column)) else "", + row.get(query_column) if pd.notna(row.get(query_column)) else "", + ), + axis=1, + ) + else: + out["query_id"] = out[query_column].apply( + lambda x: (_stable_query_id_from_text(x) if pd.notna(x) else _stable_query_id_from_text("")) + ) + + # chunk_id from chunk text + out["chunk_id"] = out[chunk_text_column].apply(lambda x: generate_chunk_id(str(x) if pd.notna(x) else "")) + + # score (coerce to float) + out["score"] = out[score_column].apply(_parse_score) + + # paragraph for matching/display + out["paragraph"] = out[chunk_text_column].where(out[chunk_text_column].notna(), "").astype(str) + + # question text for error analysis / display (copy from query/criteria column) + out["question"] = out[query_column].where(out[query_column].notna(), "").astype(str) + + # position + if position_mode == POSITION_MODE_COLUMN and position_column: + out["position"] = out[position_column].apply(lambda x: int(x) if pd.notna(x) and str(x).strip() else 1) + out["position"] = out["position"].clip(lower=1) + elif position_mode == POSITION_MODE_SORT_BY_SCORE: + out = out.sort_values(["query_id", "score"], ascending=[True, False]) + out["position"] = out.groupby("query_id").cumcount() + 1 + else: + # row_order: preserve order, assign position per query + out["position"] = out.groupby("query_id").cumcount() + 1 + + # optional document + if document_column: + out["document"] = out[document_column] + + # Keep only standard columns so loader sees a clean schema + standard_cols = [ + "query_id", + "chunk_id", + "position", + "score", + "paragraph", + "question", + ] + if document_column: + standard_cols.append("document") + out = out[standard_cols].copy() + + logger.info( + "Normalized dataset: %d rows, %d unique queries, %d unique chunks", + len(out), + out["query_id"].nunique(), + out["chunk_id"].nunique(), + ) + return out diff --git a/report_analyst/core/benchmark/error_analysis.py b/report_analyst/core/benchmark/error_analysis.py new file mode 100644 index 00000000..74e5814b --- /dev/null +++ b/report_analyst/core/benchmark/error_analysis.py @@ -0,0 +1,453 @@ +import logging +from typing import Dict, List + +import pandas as pd + +from ...models.benchmark import ( + BenchmarkDataset, + BenchmarkDatasetContent, + RetrievalResultRow, +) + +logger = logging.getLogger(__name__) + + +def build_error_analysis_dataframe( + ground_truth: BenchmarkDatasetContent, + retrieval_results: List[RetrievalResultRow], + top_k: int, +) -> pd.DataFrame: + """ + Build a per-chunk error-analysis dataframe. + + Each row corresponds to one retrieved chunk within top_k for a given question. + Columns: + - report_name + - question + - relevant_part_text + - retrieved_chunk_text + - position_in_top_k + - expert_relevance_label + - model_score + - is_really_relevant (expert label > 0) + - query_id + - chunk_id + """ + # Index ground truth by question_id and chunk_id for fast lookup + gt_by_question: Dict[str, Dict[str, float]] = {} + gt_text_by_question: Dict[str, Dict[str, str]] = {} + report_by_question: Dict[str, str] = {} + question_text_by_id: Dict[str, str] = {} + + for q in ground_truth.questions: + question_id = q.question_id + question_text_by_id[question_id] = getattr(q, "question_text", None) or getattr(q, "text", "") or "" + # Use first report_id in metadata if available + report_name = "" + if q.ground_truth_chunks: + meta = getattr(q.ground_truth_chunks[0], "metadata", None) or {} + report_name = meta.get("document") or meta.get("report") or "" + report_by_question[question_id] = report_name + + by_chunk: Dict[str, float] = {} + text_by_chunk: Dict[str, str] = {} + for chunk in q.ground_truth_chunks: + by_chunk[chunk.chunk_id] = chunk.relevance_score + text_by_chunk[chunk.chunk_id] = getattr(chunk, "text", None) or getattr(chunk, "annotation_notes", None) or "" + gt_by_question[question_id] = by_chunk + gt_text_by_question[question_id] = text_by_chunk + + rows: List[Dict] = [] + + # Group retrieval results by query_id + results_by_query: Dict[str, List[RetrievalResultRow]] = {} + for r in retrieval_results: + results_by_query.setdefault(r.query_id, []).append(r) + + for query_id, rows_for_query in results_by_query.items(): + # Sort by position and take top_k + sorted_rows = sorted(rows_for_query, key=lambda r: r.position)[:top_k] + + gt_scores = gt_by_question.get(query_id, {}) + gt_texts = gt_text_by_question.get(query_id, {}) + report_name = report_by_question.get(query_id, "") + question_text = question_text_by_id.get(query_id, "") + + for r in sorted_rows: + expert_label = gt_scores.get(r.chunk_id, 0.0) + is_really_relevant = expert_label > 0 + relevant_part_text = gt_texts.get(r.chunk_id, "") + + rows.append( + { + "report_name": report_name, + "question_id": query_id, + "question": question_text, + "relevant_part_text": relevant_part_text, + "retrieved_chunk_text": r.chunk_text or "", + "position_in_top_k": r.position, + "expert_relevance_label": expert_label, + "model_score": r.score, + "is_really_relevant": is_really_relevant, + "chunk_id": r.chunk_id, + } + ) + + df = pd.DataFrame(rows) + logger.info(f"Built error-analysis dataframe with {len(df)} rows") + return df + + +def build_error_analysis_dataframe_from_flexible( + ground_truth_dataset: BenchmarkDataset, + benchmark_dataset: BenchmarkDataset, + top_k: int, +) -> pd.DataFrame: + """ + Build a per-chunk error-analysis dataframe from flexible BenchmarkDataset objects. + + Each row corresponds to one retrieved chunk within top_k for a given question. + Columns: + - report_name + - question_id + - question (if available in ground truth) + - relevant_part_text (from ground truth) + - retrieved_chunk_text (from benchmark) + - position_in_top_k + - expert_relevance_label (from ground truth score) + - model_score (from benchmark) + - is_really_relevant (expert label > 0) + - chunk_id + """ + # Index ground truth by query_id and chunk_id + gt_by_query: Dict[str, Dict[str, Dict]] = {} # query_id -> chunk_id -> {score, text, report, question} + # Also create an index by (report, question) pair for fallback lookup + gt_by_report_question: Dict[str, Dict[str, Dict[str, Dict]]] = ( + {} + ) # report -> question -> chunk_id -> {score, text, report, question} + + for result in ground_truth_dataset.results: + query_id = result.get_query_id() + if not query_id: + continue + + chunk_id = result.get_chunk_id() + if not chunk_id: + continue + + if query_id not in gt_by_query: + gt_by_query[query_id] = {} + + # Extract data from flexible row + data = result.data + score = result.get_score() or data.get("relevance_label") or data.get("relevance_score") or 0.0 + # Try to get relevant part text - check for "Relevant" column (case-insensitive) first + # as user mentioned it should retrieve from "Relevant" column + text = None + # Check case-insensitive for "relevant" column + for key in data.keys(): + if key.lower() == "relevant": + text = data.get(key) + break + # Fallback to other common column names + if not text: + text = data.get("context") or data.get("chunk_text") or data.get("text") or "" + report = data.get("document") or data.get("report") or data.get("report_name") or "" + question = data.get("question") or "" + + gt_entry = { + "score": float(score) if score else 0.0, + "text": str(text), + "report": str(report), + "question": str(question), + } + + gt_by_query[query_id][chunk_id] = gt_entry + + # Also index by (report, question) pair for fallback lookup + # Normalize report and question to match the normalization used later + normalized_report = " ".join(str(report).strip().split()) + normalized_question = " ".join(str(question).strip().split()) + if normalized_report and normalized_question: + if normalized_report not in gt_by_report_question: + gt_by_report_question[normalized_report] = {} + if normalized_question not in gt_by_report_question[normalized_report]: + gt_by_report_question[normalized_report][normalized_question] = {} + gt_by_report_question[normalized_report][normalized_question][chunk_id] = gt_entry + + rows: List[Dict] = [] + + # Group benchmark results by query_id (report/question pair) + # This matches the evaluation logic: group by query_id, sort by similarity score, take top-K + benchmark_by_query: Dict[str, List] = {} # query_id -> [all results for this query] + for result in benchmark_dataset.results: + query_id = result.get_query_id() + if not query_id: + continue + benchmark_by_query.setdefault(query_id, []).append(result) + + # Helper function to get similarity score for ranking + def get_similarity_score(r): + data = r.data + # Use relevant_text_sim as priority for ranking (similarity between retrieved chunk and relevant part) + # Note: sim_text_relevance is NOT used here - it's an expert-annotated label, not a ranking score + sim_score = data.get("relevant_text_sim") or r.get_score() or data.get("score") or data.get("relevance_score") or 0.0 + return float(sim_score) if sim_score else 0.0 + + # Group queries by report_name, then by question + # Structure: report_name -> question -> [query_ids for that (report, question) pair] + # query_id is typically in format "report|||question" or similar + queries_by_report_and_question: Dict[str, Dict[str, List[str]]] = {} # report -> question -> [query_ids] + + for query_id, benchmark_results in benchmark_by_query.items(): + # Try to extract report/question from query_id first (common format: "report|||question") + report_name = "" + question_text = "" + + if "|||" in query_id: + parts = query_id.split("|||", 1) + report_name = parts[0] if len(parts) > 0 else "" + question_text = parts[1] if len(parts) > 1 else "" + else: + # Fallback: get from ground truth + gt_data = gt_by_query.get(query_id, {}) + first_gt = next(iter(gt_data.values())) if gt_data else {} + report_name = first_gt.get("report", "") + question_text = first_gt.get("question", "") + + # If still empty, try to get from benchmark data + if not report_name or not question_text: + sample_result = benchmark_results[0] if benchmark_results else None + if sample_result: + data = sample_result.data + if not report_name: + report_name = data.get("report") or data.get("document") or data.get("report_name") or "" + if not question_text: + question_text = data.get("question") or "" + + if not report_name or not question_text: + # Skip this pair in report/question-based path; may still be usable in a query-only fallback. + continue + + # Normalize report_name and question_text to avoid duplicates due to whitespace/formatting + # Remove leading/trailing whitespace and normalize internal whitespace + report_name = " ".join(report_name.strip().split()) + question_text = " ".join(question_text.strip().split()) + + if report_name not in queries_by_report_and_question: + queries_by_report_and_question[report_name] = {} + if question_text not in queries_by_report_and_question[report_name]: + queries_by_report_and_question[report_name][question_text] = [] + queries_by_report_and_question[report_name][question_text].append(query_id) + + # Process in order: report (outer loop) -> question (middle loop) -> top-K chunks (inner loop) + # This ensures report_name stays constant until all questions are done, + # and question stays constant until all top-K chunks are shown + processed_pairs: Dict[tuple, bool] = {} # Track which pairs have been processed + + for report_name in sorted(queries_by_report_and_question.keys()): + for question_text in sorted(queries_by_report_and_question[report_name].keys()): + pair_key = (report_name, question_text) + + # Skip if we've already processed this exact pair (safeguard against duplicates) + if pair_key in processed_pairs: + continue # Skip this pair - already processed + + processed_pairs[pair_key] = True + query_ids = queries_by_report_and_question[report_name][question_text] + + # Check if there's any ground truth for this (report, question) pair + # If no ground truth exists, skip this pair (can't evaluate without ground truth) + report_gt_data = gt_by_report_question.get(report_name, {}) + question_gt_data = report_gt_data.get(question_text, {}) + if not question_gt_data: + # No ground truth for this (report, question) pair - skip it + logger.debug( + f"Skipping (report, question) pair with no ground truth: " + f"report='{report_name}', question='{question_text[:50]}...'" + ) + continue + + # For this (report, question) pair, collect all benchmark results + all_results_for_pair = [] + for query_id in query_ids: + all_results_for_pair.extend(benchmark_by_query[query_id]) + + # Deduplicate by (chunk_id, relevant_part_id) - this allows the same chunk to appear + # multiple times if it matches different relevant parts, but prevents duplicates + # when it matches the same relevant part or has no relevant part + chunk_and_part_to_best_result: Dict[tuple, any] = {} + for result in all_results_for_pair: + chunk_id = result.get_chunk_id() or "" + if not chunk_id: + continue + + data = result.data + relevant_part_id = data.get("relevant_part_id") or "" + # Use empty string for "no relevant part" to group those together + dedup_key = (chunk_id, str(relevant_part_id)) + + sim_score = get_similarity_score(result) + if dedup_key not in chunk_and_part_to_best_result: + chunk_and_part_to_best_result[dedup_key] = result + else: + # Keep the one with higher similarity score + existing_score = get_similarity_score(chunk_and_part_to_best_result[dedup_key]) + if sim_score > existing_score: + chunk_and_part_to_best_result[dedup_key] = result + + # Sort by similarity score (descending) and take top-K + deduplicated_results = list(chunk_and_part_to_best_result.values()) + sorted_results = sorted(deduplicated_results, key=get_similarity_score, reverse=True)[:top_k] + + # Get a representative query_id for this pair (for display purposes) + query_id = query_ids[0] + + # Output all top-K chunks for this (report, question) pair + for local_rank, r in enumerate(sorted_results, start=1): + data = r.data + retrieved_chunk_id = r.get_chunk_id() or "" # This is the retrieved paragraph ID + original_position = r.get_position() or 0 + + # Get similarity score (used for ranking) + get_similarity_score(r) + + # Get model score from benchmark dataset - should be relevant_text_sim for this chunk + model_score = data.get("relevant_text_sim") or 0.0 + model_score = float(model_score) if model_score else 0.0 + + chunk_text = data.get("paragraph") or data.get("chunk_text") or data.get("text") or "" + + # Get relevant_part_id to look up ground truth + relevant_part_id_from_data = data.get("relevant_part_id") + relevant_part_id = ( + relevant_part_id_from_data + if relevant_part_id_from_data + else "" # Don't fallback to retrieved_chunk_id - if no relevant_part_id, it means no match + ) + + # Get the query_id for this specific result (not the grouped one) + result_query_id = r.get_query_id() or query_id + + # Look up ground truth using the result's query_id and relevant_part_id + # The ground truth is indexed by query_id -> chunk_id (where chunk_id is the relevant part ID) + gt_chunk = {} + if relevant_part_id: + if result_query_id: + # Primary lookup: by query_id + result_gt_data = gt_by_query.get(result_query_id, {}) + gt_chunk = result_gt_data.get(str(relevant_part_id), {}) + + # Fallback lookup: by (report, question) pair if primary lookup failed + if not gt_chunk and report_name and question_text: + report_gt_data = gt_by_report_question.get(report_name, {}) + question_gt_data = report_gt_data.get(question_text, {}) + gt_chunk = question_gt_data.get(str(relevant_part_id), {}) + + # If still no match by ID, but we have a (report, question) pair, + # get the most relevant part (highest score) for this pair (as user requested: + # "retrieve the info about relevant part from ground truth for each question/report pair") + if not gt_chunk and report_name and question_text: + report_gt_data = gt_by_report_question.get(report_name, {}) + question_gt_data = report_gt_data.get(question_text, {}) + if question_gt_data: + # Use the relevant part with the highest score for this (report, question) pair + best_chunk_id = max( + question_gt_data.keys(), + key=lambda cid: question_gt_data[cid].get("score", 0.0), + ) + gt_chunk = question_gt_data[best_chunk_id] + + relevant_part_text = gt_chunk.get("text", "") + + # Get relevance from benchmarking dataset (not ground truth) + # Check for relevance_label or relevance column in benchmark data + benchmark_relevance = data.get("relevance_label") or data.get("relevance") or data.get("label") or 0.0 + # Convert to numeric if it's a string + if isinstance(benchmark_relevance, str): + try: + benchmark_relevance = float(benchmark_relevance) + except (ValueError, TypeError): + benchmark_relevance = 0.0 + else: + benchmark_relevance = float(benchmark_relevance) if benchmark_relevance else 0.0 + + # is_really_relevant should be true only if benchmark relevance > 0 + is_really_relevant = benchmark_relevance > 0 + + rows.append( + { + "report_name": report_name, + "question_id": query_id, # Keep query_id for reference + "question": question_text, + "relevant_part_text": relevant_part_text, + "retrieved_chunk_text": str(chunk_text), + # 1..K rank within top-K for this (report, question) pair + "position_in_top_k": local_rank, + # Original retrieval rank (if available) + "retrieval_rank": original_position, + "model_score": model_score, + "is_really_relevant": is_really_relevant, + "chunk_id": retrieved_chunk_id, # Store the retrieved paragraph ID + } + ) + + # If we didn't produce any rows using the (report, question) grouping path, + # fall back to a simpler query-centric path that only relies on query_id and chunk_id. + if not rows: + for query_id, benchmark_results in benchmark_by_query.items(): + gt_chunks = gt_by_query.get(query_id, {}) + if not gt_chunks: + # No ground truth for this query_id + continue + + # Sort benchmark results by similarity score and take top-K + sorted_results = sorted(benchmark_results, key=get_similarity_score, reverse=True)[:top_k] + + for local_rank, r in enumerate(sorted_results, start=1): + data = r.data + retrieved_chunk_id = r.get_chunk_id() or "" + if not retrieved_chunk_id: + continue + + gt_entry = gt_chunks.get(retrieved_chunk_id, {}) + expert_score = float(gt_entry.get("score", 0.0)) + relevant_part_text = gt_entry.get("text", "") + report_name = ( + gt_entry.get("report") or data.get("document") or data.get("report") or data.get("report_name") or "" + ) + question_text = gt_entry.get("question") or data.get("question") or "" + + # Model score: keep same priority as main path + model_score = ( + data.get("relevant_text_sim") or r.get_score() or data.get("score") or data.get("relevance_score") or 0.0 + ) + try: + model_score = float(model_score) if model_score else 0.0 + except (ValueError, TypeError): + model_score = 0.0 + + original_position = r.get_position() or 0 + is_really_relevant = expert_score > 0 + + rows.append( + { + "report_name": str(report_name), + "question_id": query_id, + "question": str(question_text), + "relevant_part_text": str(relevant_part_text), + "retrieved_chunk_text": str(data.get("paragraph") or data.get("chunk_text") or data.get("text") or ""), + "position_in_top_k": local_rank, + "retrieval_rank": original_position, + "model_score": model_score, + "is_really_relevant": is_really_relevant, + "chunk_id": retrieved_chunk_id, + } + ) + + df = pd.DataFrame(rows) + # No need to sort - rows are already in the correct order: + # report (outer loop) -> question (middle loop) -> position_in_top_k (inner loop) + + logger.info(f"Built error-analysis dataframe from flexible datasets with {len(df)} rows") + return df diff --git a/report_analyst/core/benchmark/evaluation_engine.py b/report_analyst/core/benchmark/evaluation_engine.py new file mode 100644 index 00000000..dff9b7fe --- /dev/null +++ b/report_analyst/core/benchmark/evaluation_engine.py @@ -0,0 +1,650 @@ +import logging +from typing import Dict, List, Optional + +import numpy as np + +from ...models.benchmark import ( + BenchmarkDataset, + BenchmarkDatasetContent, + DatasetType, + EvaluationMetrics, + RetrievalConfig, + RetrievalResultsDataset, +) + +logger = logging.getLogger(__name__) + + +class EvaluationEngine: + """Engine for evaluating retrieval performance against benchmark datasets""" + + def __init__(self): + self.default_k_values = [1, 3, 5, 10] + + def evaluate_retrieval( + self, + dataset: BenchmarkDatasetContent, + retrieval_results: Dict[str, List[Dict]], + config: RetrievalConfig, + k_values: Optional[List[int]] = None, + ) -> EvaluationMetrics: + """ + Evaluate retrieval results against ground truth + + Args: + dataset: Benchmark dataset with ground truth + retrieval_results: Dict mapping question_id to list of retrieved chunks + config: Retrieval configuration used + k_values: List of K values to compute metrics for + + Returns: + EvaluationMetrics with computed scores + """ + if k_values is None: + k_values = self.default_k_values + + logger.info(f"Evaluating retrieval for {len(dataset.questions)} questions") + + # Collect results per question + question_results = [] + for question in dataset.questions: + if question.question_id not in retrieval_results: + logger.warning(f"No retrieval results for question {question.question_id}") + continue + + retrieved_chunks = retrieval_results[question.question_id] + ground_truth = {chunk.chunk_id: chunk.relevance_score for chunk in question.ground_truth_chunks} + + result = self._evaluate_single_question(retrieved_chunks, ground_truth, k_values) + question_results.append(result) + + # Aggregate metrics + return self._aggregate_metrics(question_results, k_values) + + def _evaluate_single_question( + self, + retrieved_chunks: List[Dict], + ground_truth: Dict[str, float], + k_values: List[int], + relevant_text_sim_scores: Optional[List[float]] = None, + relevance_threshold: float = 0.95, + ) -> Dict: + """Evaluate a single question's retrieval results + + Args: + retrieved_chunks: List of retrieved chunks with metadata + ground_truth: Dict mapping chunk_id to relevance score (for total count and NDCG ideal) + k_values: List of K values to compute metrics for + relevant_text_sim_scores: Optional list of relevant_text_sim scores from benchmark dataset. + If provided, used for binary relevance (relevant if > threshold). + Must match order of retrieved_chunks after deduplication. + relevance_threshold: Threshold for relevant_text_sim to consider a chunk relevant (default: 0.95) + """ + + # Extract relevant part IDs in retrieval order + # Note: "id" field contains relevant_part_id (for matching to ground truth) + # chunk_id is the unique paragraph identifier (for reference) + # Multiple retrieved paragraphs can legitimately match the same relevant part + retrieved_ids_raw = [chunk.get("id", chunk.get("chunk_id", "")) for chunk in retrieved_chunks] + + # Deduplicate relevant parts while preserving order (keep first occurrence) + # This ensures each unique relevant part is counted only once in recall/NDCG + # Multiple retrieved paragraphs matching the same relevant part is valid, but we only + # count the relevant part once to prevent recall > 1.0 + seen = set() + retrieved_ids = [] + deduplicated_indices = [] # Track which original indices were kept + for i, relevant_part_id in enumerate(retrieved_ids_raw): + if relevant_part_id not in seen: + retrieved_ids.append(relevant_part_id) + seen.add(relevant_part_id) + deduplicated_indices.append(i) + + # Get relevance scores for retrieved chunks (after deduplication) + # For NDCG, we still use ground truth scores if available + retrieved_relevance = [ground_truth.get(chunk_id, 0.0) for chunk_id in retrieved_ids] + + # Decide whether we have usable relevant_text_sim scores + use_sim_scores = bool( + relevant_text_sim_scores and any((s is not None and float(str(s) or 0) != 0.0) for s in relevant_text_sim_scores) + ) + + binary_relevance: List[int] = [] + + if use_sim_scores: + # Use relevant_text_sim threshold: chunk is relevant if relevant_text_sim > threshold (default 0.95) + # Map to deduplicated indices + for idx in deduplicated_indices: + if idx < len(relevant_text_sim_scores): + sim_score = relevant_text_sim_scores[idx] + # Convert to float if needed + if isinstance(sim_score, str): + try: + sim_score = float(sim_score) + except (ValueError, TypeError): + sim_score = 0.0 + else: + sim_score = float(sim_score) if sim_score else 0.0 + is_relevant = 1 if sim_score > relevance_threshold else 0 + binary_relevance.append(is_relevant) + else: + # Missing score, default to not relevant + binary_relevance.append(0) + else: + # Fallback: use ground truth matching (old behavior): + # treat a retrieved chunk as relevant if its chunk_id appears in the ground truth. + for chunk_id in retrieved_ids: + binary_relevance.append(1 if ground_truth.get(chunk_id, 0.0) > 0 else 0) + + # Total relevant chunks in ground truth + total_relevant = sum(1 for score in ground_truth.values() if score > 0) + + result = { + "retrieved_ids": retrieved_ids, + "retrieved_relevance": retrieved_relevance, + "binary_relevance": binary_relevance, + "total_relevant": total_relevant, + "precision_at_k": {}, + "recall_at_k": {}, + "f1_at_k": {}, + "ndcg_at_k": {}, + "reciprocal_rank": 0.0, + "average_precision": 0.0, + } + + # Compute metrics at different K values + # NOTE: Metrics should be calculated for ALL k values, even if k > len(retrieved_ids) + # When k > len(retrieved_ids), recall@k equals the maximum recall (all relevant items found) + # and precision@k is calculated with k as denominator (correctly lower when fewer items retrieved) + for k in k_values: + precision_k = self._precision_at_k(binary_relevance, k) + recall_k = self._recall_at_k(binary_relevance, total_relevant, k) + f1_k = self._f1_at_k(precision_k, recall_k) + ndcg_k = self._ndcg_at_k(retrieved_relevance, ground_truth, k) + result["precision_at_k"][k] = precision_k + result["recall_at_k"][k] = recall_k + result["f1_at_k"][k] = f1_k + result["ndcg_at_k"][k] = ndcg_k + + # Compute MRR and MAP + result["reciprocal_rank"] = self._reciprocal_rank(binary_relevance) + result["average_precision"] = self._average_precision(binary_relevance) + + return result + + def _precision_at_k(self, binary_relevance: List[int], k: int) -> float: + """Compute precision at K""" + if k == 0: + return 0.0 + relevant_at_k = sum(binary_relevance[:k]) + return relevant_at_k / k + + def _recall_at_k(self, binary_relevance: List[int], total_relevant: int, k: int) -> float: + """Compute recall at K""" + if total_relevant == 0: + return 0.0 + relevant_at_k = sum(binary_relevance[:k]) + return relevant_at_k / total_relevant + + def _f1_at_k(self, precision: float, recall: float) -> float: + """Compute F1 score from precision and recall""" + if precision + recall == 0: + return 0.0 + return 2 * (precision * recall) / (precision + recall) + + def _ndcg_at_k(self, retrieved_relevance: List[float], ground_truth: Dict[str, float], k: int) -> float: + """Compute Normalized Discounted Cumulative Gain at K""" + if k == 0: + return 0.0 + + # DCG at K + dcg = 0.0 + for i in range(min(k, len(retrieved_relevance))): + if i == 0: + dcg += retrieved_relevance[i] + else: + dcg += retrieved_relevance[i] / np.log2(i + 1) + + # Ideal DCG at K (sort ground truth scores in descending order) + ideal_scores = sorted(ground_truth.values(), reverse=True) + idcg = 0.0 + for i in range(min(k, len(ideal_scores))): + if i == 0: + idcg += ideal_scores[i] + else: + idcg += ideal_scores[i] / np.log2(i + 1) + + if idcg == 0: + return 0.0 + + return dcg / idcg + + def _reciprocal_rank(self, binary_relevance: List[int]) -> float: + """Compute reciprocal rank (1/rank of first relevant item)""" + for i, rel in enumerate(binary_relevance): + if rel == 1: + return 1.0 / (i + 1) + return 0.0 + + def _average_precision(self, binary_relevance: List[int]) -> float: + """Compute average precision""" + if not any(binary_relevance): + return 0.0 + + ap = 0.0 + relevant_count = 0 + + for i, rel in enumerate(binary_relevance): + if rel == 1: + relevant_count += 1 + precision_at_i = relevant_count / (i + 1) + ap += precision_at_i + + total_relevant = sum(binary_relevance) + return ap / total_relevant if total_relevant > 0 else 0.0 + + def _aggregate_metrics(self, question_results: List[Dict], k_values: List[int]) -> EvaluationMetrics: + """Aggregate metrics across all questions""" + if not question_results: + return EvaluationMetrics() + + metrics = EvaluationMetrics() + + # Aggregate precision, recall, F1, and NDCG at K + for k in k_values: + precisions = [r["precision_at_k"].get(k, 0.0) for r in question_results] + recalls = [r["recall_at_k"].get(k, 0.0) for r in question_results] + f1s = [r["f1_at_k"].get(k, 0.0) for r in question_results] + ndcgs = [r["ndcg_at_k"].get(k, 0.0) for r in question_results] + + metrics.precision_at_k[k] = np.mean(precisions) + metrics.recall_at_k[k] = np.mean(recalls) + metrics.f1_at_k[k] = np.mean(f1s) + metrics.ndcg_at_k[k] = np.mean(ndcgs) + + # Aggregate MRR and MAP + reciprocal_ranks = [r["reciprocal_rank"] for r in question_results] + average_precisions = [r["average_precision"] for r in question_results] + + metrics.mean_reciprocal_rank = np.mean(reciprocal_ranks) + metrics.mean_average_precision = np.mean(average_precisions) + + logger.info(f"Evaluation complete. MAP: {metrics.mean_average_precision:.3f}, MRR: {metrics.mean_reciprocal_rank:.3f}") + + return metrics + + def compare_evaluations(self, eval1: EvaluationMetrics, eval2: EvaluationMetrics) -> Dict[str, float]: + """Compare two evaluations and return improvement metrics""" + comparison = {} + + # Compare MAP and MRR + comparison["map_improvement"] = eval2.mean_average_precision - eval1.mean_average_precision + comparison["mrr_improvement"] = eval2.mean_reciprocal_rank - eval1.mean_reciprocal_rank + + # Compare metrics at K + for k in eval1.precision_at_k.keys(): + if k in eval2.precision_at_k: + comparison[f"precision_at_{k}_improvement"] = eval2.precision_at_k[k] - eval1.precision_at_k[k] + comparison[f"recall_at_{k}_improvement"] = eval2.recall_at_k[k] - eval1.recall_at_k[k] + comparison[f"f1_at_{k}_improvement"] = eval2.f1_at_k[k] - eval1.f1_at_k[k] + comparison[f"ndcg_at_{k}_improvement"] = eval2.ndcg_at_k[k] - eval1.ndcg_at_k[k] + + return comparison + + def compare_flexible_datasets( + self, + reference_dataset: BenchmarkDataset, + input_dataset: BenchmarkDataset, + k_values: Optional[List[int]] = None, + ) -> EvaluationMetrics: + """ + Compare two flexible benchmark datasets (supports both IR and IE). + + The reference dataset is treated as ground truth (e.g., "climretrieve", "chatreport"). + The input dataset contains the actual results to evaluate. + + For IR datasets: Compares retrieved chunks (by chunk_id and position). + For IE datasets: Compares answers/analysis (by query_id, comparing text similarity). + + Args: + reference_dataset: Reference dataset (ground truth) + input_dataset: Input dataset (actual results) to evaluate + k_values: List of K values to compute metrics for (only used for IR) + + Returns: + EvaluationMetrics comparing input_dataset against reference_dataset + """ + if reference_dataset.dataset_type != input_dataset.dataset_type: + logger.warning( + f"Dataset type mismatch: reference={reference_dataset.dataset_type.value}, " + f"input={input_dataset.dataset_type.value}. Attempting comparison anyway." + ) + + if reference_dataset.dataset_type == DatasetType.INFORMATION_RETRIEVAL: + return self._compare_ir_datasets(reference_dataset, input_dataset, k_values) + elif reference_dataset.dataset_type == DatasetType.INFORMATION_EXTRACTION: + return self._compare_ie_datasets(reference_dataset, input_dataset) + else: + raise ValueError(f"Unsupported dataset type: {reference_dataset.dataset_type}") + + def _compare_ir_datasets( + self, + reference_dataset: BenchmarkDataset, + input_dataset: BenchmarkDataset, + k_values: Optional[List[int]] = None, + ) -> EvaluationMetrics: + """Compare two Information Retrieval datasets""" + if k_values is None: + k_values = self.default_k_values + + # Get common queries + reference_queries = set(reference_dataset.get_unique_queries()) + input_queries = set(input_dataset.get_unique_queries()) + common_queries = reference_queries.intersection(input_queries) + + if not common_queries: + logger.warning("No common queries found between reference and input datasets") + return EvaluationMetrics() + + logger.info(f"Found {len(common_queries)} common queries for IR comparison") + + # Collect results per question + question_results = [] + for query_id in common_queries: + # Get reference results (ground truth) + reference_results = reference_dataset.get_results_by_query(query_id) + reference_results = sorted(reference_results, key=lambda x: x.get_position() or 999) + + # Build ground truth mapping: ID used for matching -> relevance_score + # IMPORTANT: Use the same identifier that will be used for retrieved_ids + # in _evaluate_single_question. That function uses the "id" field from + # retrieved_chunks, which we construct from relevant_part_id when + # available (fallback to chunk_id). To keep NDCG consistent with + # binary relevance/recall, we therefore key ground_truth by + # relevant_part_id if it exists, otherwise by chunk_id. + ground_truth = {} + for i, ref_result in enumerate(reference_results): + chunk_id = ref_result.get_chunk_id() + data = ref_result.data if hasattr(ref_result, "data") else {} + match_id = data.get("relevant_part_id") or chunk_id + if match_id: + score = ref_result.get_score() + # Use score if available, otherwise use inverse position + relevance_score = score if score is not None else max(0.0, 1.0 - (i * 0.1)) + ground_truth[match_id] = relevance_score + + # Skip this query if there's no ground truth (can't evaluate without ground truth) + if not ground_truth: + logger.debug(f"Skipping query with no ground truth: query_id='{query_id}'") + continue + + # Get input results (actual retrieval) + input_results = input_dataset.get_results_by_query(query_id) + + # Helper function to get similarity score for ranking (same logic as error analysis) + def get_similarity_score_for_ranking(result): + data = result.data if hasattr(result, "data") else {} + # Use relevant_text_sim as priority for ranking (similarity between retrieved chunk and relevant part) + # Note: sim_text_relevance is NOT used here - it's an expert-annotated label, not a ranking score + sim_score = ( + data.get("relevant_text_sim") + or result.get_score() + or data.get("score") + or data.get("relevance_score") + or 0.0 + ) + return float(sim_score) if sim_score else 0.0 + + # Sort by similarity score (descending) - this matches the error analysis logic + input_results = sorted(input_results, key=get_similarity_score_for_ranking, reverse=True) + + # Convert to format expected by _evaluate_single_question + # Use relevant_part_id for matching to ground truth (if available), otherwise fallback to chunk_id + retrieved_chunks = [] + relevant_text_sim_scores = [] # Extract relevant_text_sim from benchmark dataset + for result in input_results: + chunk_id = result.get_chunk_id() # Unique paragraph identifier + relevant_part_id = result.get("relevant_part_id") # For matching to ground truth + # Fallback: if no relevant_part_id, use chunk_id (backward compatibility) + match_id = relevant_part_id if relevant_part_id else chunk_id + # Use relevant_text_sim as the score (same priority as ranking) + data = result.data if hasattr(result, "data") else {} + score = ( + data.get("relevant_text_sim") + or result.get_score() + or data.get("score") + or data.get("relevance_score") + or 0.0 + ) + score = float(score) if score else 0.0 + position = result.get_position() or 999 + + # Extract relevant_text_sim for relevance determination (chunk is relevant if relevant_text_sim > 0.95) + relevant_text_sim = data.get("relevant_text_sim") or 0.0 + # Convert to numeric if it's a string + if isinstance(relevant_text_sim, str): + try: + relevant_text_sim = float(relevant_text_sim) + except (ValueError, TypeError): + relevant_text_sim = 0.0 + else: + relevant_text_sim = float(relevant_text_sim) if relevant_text_sim else 0.0 + relevant_text_sim_scores.append(relevant_text_sim) + + chunk_dict = { + "id": match_id or f"unknown_{position}", # Use relevant_part_id for matching + "chunk_id": chunk_id or f"unknown_{position}", # Keep original chunk_id for reference + "relevant_part_id": match_id or f"unknown_{position}", # Store for logging + "score": score, + "position": position, + } + retrieved_chunks.append(chunk_dict) + + # Evaluate this question + result = self._evaluate_single_question( + retrieved_chunks, + ground_truth, + k_values, + relevant_text_sim_scores, + relevance_threshold=0.95, + ) + question_results.append(result) + + # Aggregate metrics + return self._aggregate_metrics(question_results, k_values) + + def _compare_ie_datasets(self, reference_dataset: BenchmarkDataset, input_dataset: BenchmarkDataset) -> EvaluationMetrics: + """ + Compare two Information Extraction datasets. + + Compares answers/analysis text using similarity metrics. + For structured fields (categories, extracted values), uses exact match. + """ + # Get common queries + reference_queries = set(reference_dataset.get_unique_queries()) + input_queries = set(input_dataset.get_unique_queries()) + common_queries = reference_queries.intersection(input_queries) + + if not common_queries: + logger.warning("No common queries found between reference and input datasets") + return EvaluationMetrics() + + logger.info(f"Found {len(common_queries)} common queries for IE comparison") + + # For IE, we compare answers/analysis per query + # Metrics: Exact match, F1 (token-level), BLEU, ROUGE, etc. + exact_matches = 0 + total_queries = len(common_queries) + + # Simple comparison: exact match for now + # TODO: Add more sophisticated metrics (F1, BLEU, ROUGE, semantic similarity) + for query_id in common_queries: + reference_results = reference_dataset.get_results_by_query(query_id) + input_results = input_dataset.get_results_by_query(query_id) + + if not reference_results or not input_results: + continue + + # Get first result for each (assuming one answer per query) + ref_answer = reference_results[0].get_answer() + input_answer = input_results[0].get_answer() + + if ref_answer and input_answer: + # Normalize and compare + ref_normalized = ref_answer.strip().lower() + input_normalized = input_answer.strip().lower() + + if ref_normalized == input_normalized: + exact_matches += 1 + + # Create metrics (simplified for IE) + metrics = EvaluationMetrics() + if total_queries > 0: + exact_match_rate = exact_matches / total_queries + # For IE, we use exact_match_rate as the primary metric + # Map to precision@1 for consistency with IR metrics + metrics.precision_at_k[1] = exact_match_rate + metrics.mean_average_precision = exact_match_rate + metrics.mean_reciprocal_rank = exact_match_rate if exact_matches > 0 else 0.0 + + logger.info( + "IE comparison complete. Exact match rate: " + f"{exact_matches}/{total_queries} = {metrics.precision_at_k.get(1, 0.0):.3f}" + ) + + return metrics + + def compare_datasets( + self, + reference_dataset: RetrievalResultsDataset, + input_dataset: RetrievalResultsDataset, + k_values: Optional[List[int]] = None, + match_on_chunk_id: bool = True, + ) -> EvaluationMetrics: + """ + Compare two retrieval results datasets. + + The reference dataset is treated as ground truth (e.g., "climretrieve"). + The input dataset contains the actual retrieval results to evaluate. + + Args: + reference_dataset: Reference dataset (ground truth) - e.g., climretrieve + input_dataset: Input dataset (actual retrieval results) to evaluate + k_values: List of K values to compute metrics for + match_on_chunk_id: If True, match chunks by chunk_id. If False, match by position only. + + Returns: + EvaluationMetrics comparing input_dataset against reference_dataset + """ + if k_values is None: + k_values = self.default_k_values + + logger.info(f"Comparing datasets: reference='{reference_dataset.dataset_id}' vs input='{input_dataset.dataset_id}'") + + # Get common queries + reference_queries = set(reference_dataset.get_unique_queries()) + input_queries = set(input_dataset.get_unique_queries()) + common_queries = reference_queries.intersection(input_queries) + + if not common_queries: + logger.warning("No common queries found between reference and input datasets") + return EvaluationMetrics() + + logger.info(f"Found {len(common_queries)} common queries") + + # Collect results per question + question_results = [] + for query_id in common_queries: + # Get reference results (ground truth) + reference_results = reference_dataset.get_results_by_query(query_id) + # Sort by position to get ground truth order + reference_results = sorted(reference_results, key=lambda x: x.position) + + # Build ground truth mapping: chunk_id -> relevance_score + # Use position as relevance score (higher position = lower relevance) + # Or use actual score if available + ground_truth = {} + for i, ref_result in enumerate(reference_results): + chunk_id = ref_result.chunk_id + # Use score if available, otherwise use inverse position (1.0 for position 1, 0.9 for position 2, etc.) + relevance_score = ref_result.score if ref_result.score > 0 else max(0.0, 1.0 - (i * 0.1)) + ground_truth[chunk_id] = relevance_score + + # Skip this query if there's no ground truth (can't evaluate without ground truth) + if not ground_truth: + logger.debug(f"Skipping query with no ground truth: query_id='{query_id}'") + continue + + # Get input results (actual retrieval) + input_results = input_dataset.get_results_by_query(query_id) + + # Helper function to get similarity score for ranking + def get_similarity_score_for_ranking_legacy(result): + # For legacy RetrievalResultRow, check if it has data attribute + if hasattr(result, "metadata") and result.metadata: + data = result.metadata + # Use relevant_text_sim as priority for ranking + sim_score = ( + data.get("relevant_text_sim") + or result.score + or data.get("score") + or data.get("relevance_score") + or 0.0 + ) + return float(sim_score) if sim_score else 0.0 + # Fallback to score if no metadata + return float(result.score) if result.score else 0.0 + + # Sort by similarity score (descending) - prioritize relevant_text_sim + input_results = sorted(input_results, key=get_similarity_score_for_ranking_legacy, reverse=True) + + # Convert to format expected by _evaluate_single_question + retrieved_chunks = [] + relevant_text_sim_scores = [] # Extract relevant_text_sim from benchmark dataset + for result in input_results: + # Use relevant_text_sim as the score if available + score = result.score or 0.0 + relevant_text_sim = 0.0 + if hasattr(result, "metadata") and result.metadata: + data = result.metadata + score = ( + data.get("relevant_text_sim") + or result.score + or data.get("score") + or data.get("relevance_score") + or 0.0 + ) + # Extract relevant_text_sim for relevance determination (chunk is relevant if relevant_text_sim > 0.95) + relevant_text_sim = data.get("relevant_text_sim") or 0.0 + score = float(score) if score else 0.0 + + # Convert to numeric if it's a string + if isinstance(relevant_text_sim, str): + try: + relevant_text_sim = float(relevant_text_sim) + except (ValueError, TypeError): + relevant_text_sim = 0.0 + else: + relevant_text_sim = float(relevant_text_sim) if relevant_text_sim else 0.0 + relevant_text_sim_scores.append(relevant_text_sim) + + chunk_dict = { + "id": result.chunk_id, + "chunk_id": result.chunk_id, + "score": score, + "position": result.position, + } + retrieved_chunks.append(chunk_dict) + + # Evaluate this question + result = self._evaluate_single_question( + retrieved_chunks, + ground_truth, + k_values, + relevant_text_sim_scores, + relevance_threshold=0.95, + ) + question_results.append(result) + + # Aggregate metrics + return self._aggregate_metrics(question_results, k_values) diff --git a/report_analyst/core/benchmark/flexible_alignment.py b/report_analyst/core/benchmark/flexible_alignment.py new file mode 100644 index 00000000..072170d3 --- /dev/null +++ b/report_analyst/core/benchmark/flexible_alignment.py @@ -0,0 +1,240 @@ +import logging +from dataclasses import dataclass +from typing import Any, List, Optional + +import pandas as pd + +from .dataset_normalizer import ( + make_chunk_id_from_text, + make_query_id_from_columns, + make_relevant_part_id_from_text, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class GroundTruthAlignConfig: + document_col: Optional[str] + question_col: str + chunk_text_col: str + relevant_part_col: Optional[str] + label_cols: List[str] + + +@dataclass +class BenchmarkAlignConfig: + document_col: Optional[str] + question_col: Optional[str] + query_id_col: Optional[str] + chunk_text_col: str + relevant_part_col: Optional[str] + prediction_cols: List[str] + ranking_score_col: Optional[str] + + +def align_ground_truth_flexible(df: pd.DataFrame, config: GroundTruthAlignConfig) -> pd.DataFrame: + """ + Align a raw ground truth DataFrame to the unified flexible schema. + + Output columns: + - query_id + - chunk_id + - relevant_part_id (optional) + - question + - document (optional) + - chunk_text + - relevant_part_text (optional) + - plus all label columns from config.label_cols + - score (numeric) for ranking, derived from the first label column when present + """ + if config.question_col not in df.columns: + raise ValueError(f"Question/description column '{config.question_col}' not in DataFrame.") + if config.chunk_text_col not in df.columns: + raise ValueError(f"Chunk text column '{config.chunk_text_col}' not in DataFrame.") + + for col in config.label_cols: + if col not in df.columns: + raise ValueError(f"Label column '{col}' not in DataFrame.") + + if config.document_col and config.document_col not in df.columns: + raise ValueError(f"Document column '{config.document_col}' not in DataFrame.") + if config.relevant_part_col and config.relevant_part_col not in df.columns: + raise ValueError(f"Relevant part column '{config.relevant_part_col}' not in DataFrame.") + + out = df.copy() + + # Core ID fields + def _mk_qid(row): + doc_val = row[config.document_col] if config.document_col else None # type: ignore[index] + return make_query_id_from_columns(doc_val, row[config.question_col]) + + out["query_id"] = out.apply(_mk_qid, axis=1) + out["chunk_id"] = out[config.chunk_text_col].apply(make_chunk_id_from_text) + + if config.relevant_part_col: + out["relevant_part_text"] = out[config.relevant_part_col] + out["relevant_part_id"] = out[config.relevant_part_col].apply(make_relevant_part_id_from_text) + else: + out["relevant_part_text"] = "" + # Fallback: use chunk_id when no explicit relevant part is provided + out["relevant_part_id"] = out["chunk_id"] + + # Human-readable text fields + out["question"] = out[config.question_col].astype(str) + if config.document_col: + out["document"] = out[config.document_col].astype(str) + else: + out["document"] = "" + out["chunk_text"] = out[config.chunk_text_col].astype(str) + + # Preserve label columns as-is + for col in config.label_cols: + out[col] = out[col] + + # Provide a numeric "score" column for ranking based on the first label column, if any + if config.label_cols: + primary_label = config.label_cols[0] + + def _parse_score(val: Any) -> float: + try: + return float(str(val).strip()) + except (TypeError, ValueError): + return 0.0 + + out["score"] = out[primary_label].apply(_parse_score) + + # Select and order columns + base_cols: List[str] = [ + "query_id", + "chunk_id", + "relevant_part_id", + "question", + "document", + "chunk_text", + "relevant_part_text", + ] + label_cols = [c for c in config.label_cols if c in out.columns] + extra_cols = [c for c in ["score"] if c in out.columns] + ordered_cols = base_cols + label_cols + extra_cols + + out_aligned = out[ordered_cols].copy() + + logger.info( + "Aligned ground truth (flexible): %d rows, %d unique queries, %d unique chunks", + len(out_aligned), + out_aligned["query_id"].nunique(), + out_aligned["chunk_id"].nunique(), + ) + return out_aligned + + +def align_benchmark_flexible(df: pd.DataFrame, config: BenchmarkAlignConfig) -> pd.DataFrame: + """ + Align a raw benchmark results DataFrame to the unified flexible schema. + + Output columns: + - query_id + - chunk_id + - relevant_part_id (optional) + - chunk_text + - relevant_part_text_pred (optional) + - plus all prediction columns from config.prediction_cols + - relevant_text_sim (optional) duplicated from ranking_score_col for reuse by ranking logic + """ + if config.query_id_col: + if config.query_id_col not in df.columns: + raise ValueError(f"query_id column '{config.query_id_col}' not in DataFrame.") + else: + # If no explicit query_id column, require at least a question/description column + if not config.question_col or config.question_col not in df.columns: + raise ValueError("Either an explicit query_id_col or a question/description column must be provided.") + + if config.chunk_text_col not in df.columns: + raise ValueError(f"Chunk text column '{config.chunk_text_col}' not in DataFrame.") + + for col in config.prediction_cols: + if col not in df.columns: + raise ValueError(f"Prediction column '{col}' not in DataFrame.") + + if config.document_col and config.document_col not in df.columns: + raise ValueError(f"Document column '{config.document_col}' not in DataFrame.") + if config.relevant_part_col and config.relevant_part_col not in df.columns: + raise ValueError(f"Relevant part column '{config.relevant_part_col}' not in DataFrame.") + if config.ranking_score_col and config.ranking_score_col not in df.columns: + raise ValueError(f"Ranking score column '{config.ranking_score_col}' not in DataFrame.") + + out = df.copy() + + # query_id: either reuse existing column or recompute from document/question + if config.query_id_col: + out["query_id"] = out[config.query_id_col].astype(str) + else: + + def _mk_qid(row): + doc_val = row[config.document_col] if config.document_col else None # type: ignore[index] + return make_query_id_from_columns(doc_val, row[config.question_col]) # type: ignore[index] + + out["query_id"] = out.apply(_mk_qid, axis=1) + + # chunk_id from chunk text + out["chunk_id"] = out[config.chunk_text_col].apply(make_chunk_id_from_text) + out["chunk_text"] = out[config.chunk_text_col].astype(str) + + # Optional predicted relevant part + if config.relevant_part_col: + out["relevant_part_text_pred"] = out[config.relevant_part_col].astype(str) + out["relevant_part_id"] = out[config.relevant_part_col].apply(make_relevant_part_id_from_text) + else: + out["relevant_part_text_pred"] = "" + out["relevant_part_id"] = out["chunk_id"] + + # Predictions / scores + for col in config.prediction_cols: + out[col] = out[col] + + # Optional ranking score column: duplicate into relevant_text_sim for reuse by ranking logic + if config.ranking_score_col: + out["relevant_text_sim"] = out[config.ranking_score_col] + + base_cols: List[str] = [ + "query_id", + "chunk_id", + "relevant_part_id", + "chunk_text", + "relevant_part_text_pred", + ] + pred_cols = [c for c in config.prediction_cols if c in out.columns] + extra_cols: List[str] = [] + if "relevant_text_sim" in out.columns: + extra_cols.append("relevant_text_sim") + + # Build final column order and remove duplicates while preserving order. + # This is important when the chosen ranking_score_col is also one of the + # prediction columns (e.g. 'relevant_text_sim'), to avoid duplicated + # column names in the aligned DataFrame. + ordered_cols_raw = base_cols + pred_cols + extra_cols + seen_cols: set[str] = set() + ordered_cols: List[str] = [] + for col in ordered_cols_raw: + if col not in seen_cols: + seen_cols.add(col) + ordered_cols.append(col) + + out_aligned = out[ordered_cols].copy() + + logger.info( + "Aligned benchmark (flexible): %d rows, %d unique queries, %d unique chunks", + len(out_aligned), + out_aligned["query_id"].nunique(), + out_aligned["chunk_id"].nunique(), + ) + return out_aligned + + +__all__ = [ + "BenchmarkAlignConfig", + "GroundTruthAlignConfig", + "align_benchmark_flexible", + "align_ground_truth_flexible", +] diff --git a/report_analyst/core/benchmark/retrieval_results_loader.py b/report_analyst/core/benchmark/retrieval_results_loader.py new file mode 100644 index 00000000..577f0a91 --- /dev/null +++ b/report_analyst/core/benchmark/retrieval_results_loader.py @@ -0,0 +1,609 @@ +""" +Loader for retrieval results datasets from CSV or SQLite. + +This module provides functions to load retrieval results in CSV format, +which can be used for benchmarking. The functions are designed to be +webhook-ready and can be used in API endpoints. +""" + +import logging +import sqlite3 +from io import StringIO +from pathlib import Path +from typing import Dict, List, Optional, Union + +import pandas as pd + +from ...models.benchmark import ( + BenchmarkDataset, + DatasetType, + FlexibleDatasetRow, + RetrievalResultRow, + RetrievalResultsDataset, +) + +logger = logging.getLogger(__name__) + + +def load_retrieval_results_from_csv( + csv_path: Optional[str] = None, + csv_content: Optional[Union[str, bytes]] = None, + dataset_id: Optional[str] = None, + dataset_name: Optional[str] = None, +) -> RetrievalResultsDataset: + """ + Load retrieval results from CSV file or content. + + This function is webhook-ready and can accept either a file path or CSV content. + + Expected CSV format: + - query_id: Question/query identifier (required) + - report_id: Report/document identifier (optional) + - chunk_id: Chunk identifier (required) + - chunk_text: Chunk text content (optional) + - position: Position/rank in retrieval (required, 1-indexed) + - score: Retrieval score (required) + - similarity_score: Vector similarity score (optional) + - llm_score: LLM-based relevance score (optional) + + Args: + csv_path: Path to CSV file (if loading from file) + csv_content: CSV content as string or bytes (if loading from upload/webhook) + dataset_id: Optional dataset identifier (auto-generated if not provided) + dataset_name: Optional dataset name (defaults to filename or 'uploaded_dataset') + + Returns: + RetrievalResultsDataset with loaded results + + Raises: + ValueError: If neither csv_path nor csv_content is provided + FileNotFoundError: If csv_path is provided but file doesn't exist + """ + if csv_path is None and csv_content is None: + raise ValueError("Either csv_path or csv_content must be provided") + + # Determine source info + if csv_path: + source_path = csv_path + source_name = Path(csv_path).stem if not dataset_name else dataset_name + if not Path(csv_path).exists(): + raise FileNotFoundError(f"CSV file not found: {csv_path}") + else: + source_path = None + source_name = dataset_name or "uploaded_dataset" + + dataset_id = dataset_id or f"{source_name}_{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}" + + # Load CSV + if csv_path: + df = pd.read_csv(csv_path) + else: + # Handle both string and bytes + if isinstance(csv_content, bytes): + csv_content = csv_content.decode("utf-8") + df = pd.read_csv(StringIO(csv_content)) + + # Validate required columns + required_columns = ["query_id", "chunk_id", "position", "score"] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + raise ValueError(f"Missing required columns: {missing_columns}") + + # Convert to RetrievalResultRow objects + results = [] + for _, row in df.iterrows(): + result_row = RetrievalResultRow( + query_id=str(row["query_id"]), + report_id=(str(row["report_id"]) if "report_id" in df.columns and pd.notna(row.get("report_id")) else None), + chunk_id=str(row["chunk_id"]), + chunk_text=(str(row["chunk_text"]) if "chunk_text" in df.columns and pd.notna(row.get("chunk_text")) else None), + position=int(row["position"]), + score=float(row["score"]), + similarity_score=( + float(row["similarity_score"]) + if "similarity_score" in df.columns and pd.notna(row.get("similarity_score")) + else None + ), + llm_score=(float(row["llm_score"]) if "llm_score" in df.columns and pd.notna(row.get("llm_score")) else None), + metadata={ + k: v + for k, v in row.items() + if k + not in [ + "query_id", + "report_id", + "chunk_id", + "chunk_text", + "position", + "score", + "similarity_score", + "llm_score", + ] + and pd.notna(v) + }, + ) + results.append(result_row) + + logger.info(f"Loaded {len(results)} retrieval results from CSV for dataset '{dataset_id}'") + + return RetrievalResultsDataset( + dataset_id=dataset_id, + name=source_name, + description="Retrieval results loaded from CSV", + source="csv", + source_path=source_path, + results=results, + ) + + +def load_retrieval_results_from_sqlite( + db_path: str, + table_name: str = "retrieval_results", + dataset_id: Optional[str] = None, + dataset_name: Optional[str] = None, + query_filter: Optional[str] = None, +) -> RetrievalResultsDataset: + """ + Load retrieval results from SQLite database. + + Expected table schema: + - query_id: TEXT (required) + - report_id: TEXT (optional) + - chunk_id: TEXT (required) + - chunk_text: TEXT (optional) + - position: INTEGER (required, 1-indexed) + - score: REAL (required) + - similarity_score: REAL (optional) + - llm_score: REAL (optional) + + Args: + db_path: Path to SQLite database + table_name: Name of the table containing retrieval results + dataset_id: Optional dataset identifier + dataset_name: Optional dataset name + query_filter: Optional SQL WHERE clause to filter results + + Returns: + RetrievalResultsDataset with loaded results + + Raises: + FileNotFoundError: If database file doesn't exist + sqlite3.Error: If database query fails + """ + if not Path(db_path).exists(): + raise FileNotFoundError(f"Database file not found: {db_path}") + + dataset_id = dataset_id or f"{Path(db_path).stem}_{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}" + dataset_name = dataset_name or Path(db_path).stem + + # Build query + base_query = f"SELECT * FROM {table_name}" # noqa: S608 + if query_filter: + base_query += f" WHERE {query_filter}" + base_query += " ORDER BY query_id, position" + + # Load data + conn = sqlite3.connect(db_path) + try: + df = pd.read_sql_query(base_query, conn) + finally: + conn.close() + + # Validate required columns + required_columns = ["query_id", "chunk_id", "position", "score"] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + raise ValueError(f"Missing required columns in database table: {missing_columns}") + + # Convert to RetrievalResultRow objects + results = [] + for _, row in df.iterrows(): + result_row = RetrievalResultRow( + query_id=str(row["query_id"]), + report_id=(str(row["report_id"]) if "report_id" in df.columns and pd.notna(row.get("report_id")) else None), + chunk_id=str(row["chunk_id"]), + chunk_text=(str(row["chunk_text"]) if "chunk_text" in df.columns and pd.notna(row.get("chunk_text")) else None), + position=int(row["position"]), + score=float(row["score"]), + similarity_score=( + float(row["similarity_score"]) + if "similarity_score" in df.columns and pd.notna(row.get("similarity_score")) + else None + ), + llm_score=(float(row["llm_score"]) if "llm_score" in df.columns and pd.notna(row.get("llm_score")) else None), + metadata={ + k: v + for k, v in row.items() + if k + not in [ + "query_id", + "report_id", + "chunk_id", + "chunk_text", + "position", + "score", + "similarity_score", + "llm_score", + ] + and pd.notna(v) + }, + ) + results.append(result_row) + + logger.info(f"Loaded {len(results)} retrieval results from SQLite for dataset '{dataset_id}'") + + return RetrievalResultsDataset( + dataset_id=dataset_id, + name=dataset_name, + description="Retrieval results loaded from SQLite database", + source="sqlite", + source_path=db_path, + results=results, + ) + + +def export_retrieval_results_to_csv(dataset: BenchmarkDataset, output_path: str) -> str: + """ + Export benchmark dataset to CSV file. + + Args: + dataset: BenchmarkDataset to export + output_path: Path where CSV file should be saved + + Returns: + Path to the saved CSV file + """ + rows = [] + for result in dataset.results: + row = { + "query_id": result.get_query_id() or "", + "report_id": result.get("report_id") or result.get("document_id") or "", + "chunk_id": result.get_chunk_id() or "", + "chunk_text": result.get("chunk_text") or "", + "position": result.get_position() or "", + "score": result.get_score() or "", + "similarity_score": result.get("similarity_score") or "", + "llm_score": result.get("llm_score") or "", + } + # Add all other fields from data dict + for key, value in result.data.items(): + if key not in row: + row[key] = value if value is not None else "" + rows.append(row) + + df = pd.DataFrame(rows) + df.to_csv(output_path, index=False) + + logger.info(f"Exported {len(rows)} retrieval results to CSV: {output_path}") + return output_path + + +def detect_dataset_type(columns: List[str]) -> DatasetType: + """ + Detect dataset type based on column names. + + Information Retrieval (IR) datasets typically have: + - chunk_id, position, score, similarity_score + + Information Extraction (IE) datasets typically have: + - answer, analysis, category, confidence_score + + Args: + columns: List of column names in the dataset + + Returns: + DatasetType enum value + """ + columns_lower = [c.lower() for c in columns] + + # Check for IE indicators + ie_indicators = [ + "answer", + "analysis", + "response", + "category", + "class", + "label", + "extracted_value", + ] + has_ie_indicators = any(indicator in columns_lower for indicator in ie_indicators) + + # Check for IR indicators + ir_indicators = [ + "chunk_id", + "chunk", + "position", + "rank", + "similarity_score", + "relevance_score", + ] + has_ir_indicators = any(indicator in columns_lower for indicator in ir_indicators) + + # If both present, prioritize IE (more specific) + if has_ie_indicators: + return DatasetType.INFORMATION_EXTRACTION + elif has_ir_indicators: + return DatasetType.INFORMATION_RETRIEVAL + else: + # Default to IR if unclear + return DatasetType.INFORMATION_RETRIEVAL + + +def load_flexible_dataset_from_csv( + csv_path: Optional[str] = None, + csv_content: Optional[Union[str, bytes]] = None, + dataset_id: Optional[str] = None, + dataset_name: Optional[str] = None, + dataset_type: Optional[DatasetType] = None, + column_mapping: Optional[Dict[str, str]] = None, +) -> BenchmarkDataset: + """ + Load a flexible benchmark dataset from CSV with automatic column name detection. + + This function is webhook-ready and handles different column name variations. + It automatically detects dataset type (IR vs IE) and maps columns flexibly. + + Args: + csv_path: Path to CSV file (if loading from file) + csv_content: CSV content as string or bytes (if loading from upload/webhook) + dataset_id: Optional dataset identifier (auto-generated if not provided) + dataset_name: Optional dataset name + dataset_type: Optional dataset type (auto-detected if not provided) + column_mapping: Optional explicit column mapping (standard_name -> actual_name) + + Returns: + BenchmarkDataset with loaded results + + Raises: + ValueError: If neither csv_path nor csv_content is provided + FileNotFoundError: If csv_path is provided but file doesn't exist + """ + if csv_path is None and csv_content is None: + raise ValueError("Either csv_path or csv_content must be provided") + + # Determine source info + if csv_path: + source_path = csv_path + source_name = Path(csv_path).stem if not dataset_name else dataset_name + if not Path(csv_path).exists(): + raise FileNotFoundError(f"CSV file not found: {csv_path}") + else: + source_path = None + source_name = dataset_name or "uploaded_dataset" + + dataset_id = dataset_id or f"{source_name}_{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}" + + # Load CSV + if csv_path: + df = pd.read_csv(csv_path) + else: + # Handle both string and bytes + if isinstance(csv_content, bytes): + csv_content = csv_content.decode("utf-8") + df = pd.read_csv(StringIO(csv_content)) + + # Detect dataset type if not provided + if dataset_type is None: + dataset_type = detect_dataset_type(df.columns.tolist()) + logger.info(f"Auto-detected dataset type: {dataset_type.value}") + + # Validate required columns + columns_lower = [c.lower() for c in df.columns] + + # Check for query/question ID (required for all datasets) + has_query_id = any(col in ["query_id", "question_id", "qid", "query"] for col in columns_lower) + if not has_query_id: + raise ValueError("Missing required columns: query_id or question_id") + + # For IR datasets, check for chunk_id, position, and score + if dataset_type == DatasetType.INFORMATION_RETRIEVAL: + has_chunk_id = any(col in ["chunk_id", "chunk", "cid"] for col in columns_lower) + if not has_chunk_id: + raise ValueError("Missing required columns: chunk_id (required for IR datasets)") + has_position = any(col in ["position", "rank", "order", "pos"] for col in columns_lower) + if not has_position: + raise ValueError("Missing required columns: position or rank (required for IR datasets)") + has_score = any(col in ["score", "relevance_score", "confidence_score", "similarity_score"] for col in columns_lower) + if not has_score: + raise ValueError("Missing required columns: score (required for IR datasets)") + + # Build column mapping if not provided + if column_mapping is None: + column_mapping = {} + # Map common variations to standard names + standard_mappings = { + "query_id": ["query_id", "question_id", "qid", "query"], + "chunk_id": ["chunk_id", "chunk", "cid"], + "report_id": ["report_id", "document_id", "doc_id", "report"], + "position": ["position", "rank", "order", "pos"], + "score": [ + "score", + "relevance_score", + "confidence_score", + "similarity_score", + ], + "answer": ["answer", "analysis", "response", "text"], + "category": ["category", "class", "label", "type"], + } + + columns_lower_dict = {c.lower(): c for c in df.columns} + for standard_name, variations in standard_mappings.items(): + for variation in variations: + if variation.lower() in columns_lower_dict: + column_mapping[standard_name] = columns_lower_dict[variation.lower()] + break + + # Convert to FlexibleDatasetRow objects + results = [] + for _, row in df.iterrows(): + # Convert row to dictionary, handling NaN values + row_dict = {} + for col in df.columns: + value = row[col] + # Convert NaN to None, keep other values as-is + if pd.isna(value): + row_dict[col] = None + else: + row_dict[col] = value + + result_row = FlexibleDatasetRow(data=row_dict) + results.append(result_row) + + logger.info(f"Loaded {len(results)} rows from CSV for dataset '{dataset_id}' (type: {dataset_type.value})") + + return BenchmarkDataset( + dataset_id=dataset_id, + name=source_name, + description=f"Flexible dataset loaded from CSV (type: {dataset_type.value})", + version="1.0", # Default version + question_set=None, # Can be inferred or set explicitly + dataset_type=dataset_type, + source="csv", + source_path=source_path, + column_mapping=column_mapping, + results=results, + ) + + +def load_flexible_dataset_from_normalized_df( + normalized_df: pd.DataFrame, + dataset_id: Optional[str] = None, + dataset_name: Optional[str] = None, +) -> BenchmarkDataset: + """ + Build a BenchmarkDataset from a DataFrame that already has standard columns + (query_id, chunk_id, position, score, and optionally paragraph, document). + Used after normalize_dataframe_for_benchmark() so the evaluation pipeline sees consistent schema. + """ + required = ["query_id", "chunk_id", "position", "score"] + missing = [c for c in required if c not in normalized_df.columns] + if missing: + raise ValueError(f"Normalized DataFrame must have columns {required}. Missing: {missing}") + source_name = dataset_name or "normalized_dataset" + did = dataset_id or f"{source_name}_{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}" + results = [] + for _, row in normalized_df.iterrows(): + row_dict = {} + for col in normalized_df.columns: + val = row[col] + row_dict[col] = None if pd.isna(val) else val + results.append(FlexibleDatasetRow(data=row_dict)) + logger.info( + "Loaded %d rows from normalized DataFrame for dataset '%s' (type: IR)", + len(results), + did, + ) + return BenchmarkDataset( + dataset_id=did, + name=source_name, + description="Flexible dataset from normalized column mapping (type: information_retrieval)", + version="1.0", + question_set=None, + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + source="csv", + source_path=None, + column_mapping={c: c for c in normalized_df.columns}, + results=results, + ) + + +def load_flexible_dataset_from_sqlite( + db_path: str, + table_name: str = "benchmark_results", + dataset_id: Optional[str] = None, + dataset_name: Optional[str] = None, + dataset_type: Optional[DatasetType] = None, + column_mapping: Optional[Dict[str, str]] = None, + query_filter: Optional[str] = None, +) -> BenchmarkDataset: + """ + Load a flexible benchmark dataset from SQLite database. + + Args: + db_path: Path to SQLite database + table_name: Name of the table containing results + dataset_id: Optional dataset identifier + dataset_name: Optional dataset name + dataset_type: Optional dataset type (auto-detected if not provided) + column_mapping: Optional explicit column mapping + query_filter: Optional SQL WHERE clause to filter results + + Returns: + BenchmarkDataset with loaded results + """ + if not Path(db_path).exists(): + raise FileNotFoundError(f"Database file not found: {db_path}") + + dataset_id = dataset_id or f"{Path(db_path).stem}_{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}" + dataset_name = dataset_name or Path(db_path).stem + + # Build query + base_query = f"SELECT * FROM {table_name}" # noqa: S608 + if query_filter: + base_query += f" WHERE {query_filter}" + + # Load data + conn = sqlite3.connect(db_path) + try: + df = pd.read_sql_query(base_query, conn) + finally: + conn.close() + + # Detect dataset type if not provided + if dataset_type is None: + dataset_type = detect_dataset_type(df.columns.tolist()) + logger.info(f"Auto-detected dataset type: {dataset_type.value}") + + # Build column mapping if not provided (same logic as CSV loader) + if column_mapping is None: + column_mapping = {} + standard_mappings = { + "query_id": ["query_id", "question_id", "qid", "query"], + "chunk_id": ["chunk_id", "chunk", "cid"], + "report_id": ["report_id", "document_id", "doc_id", "report"], + "position": ["position", "rank", "order", "pos"], + "score": [ + "score", + "relevance_score", + "confidence_score", + "similarity_score", + ], + "answer": ["answer", "analysis", "response", "text"], + "category": ["category", "class", "label", "type"], + } + + columns_lower = {c.lower(): c for c in df.columns} + for standard_name, variations in standard_mappings.items(): + for variation in variations: + if variation.lower() in columns_lower: + column_mapping[standard_name] = columns_lower[variation.lower()] + break + + # Convert to FlexibleDatasetRow objects + results = [] + for _, row in df.iterrows(): + row_dict = {} + for col in df.columns: + value = row[col] + if pd.isna(value): + row_dict[col] = None + else: + row_dict[col] = value + + result_row = FlexibleDatasetRow(data=row_dict) + results.append(result_row) + + logger.info(f"Loaded {len(results)} rows from SQLite for dataset '{dataset_id}' (type: {dataset_type.value})") + + return BenchmarkDataset( + dataset_id=dataset_id, + name=dataset_name, + description=f"Flexible dataset loaded from SQLite (type: {dataset_type.value})", + version="1.0", # Default version + question_set=None, # Can be inferred or set explicitly + dataset_type=dataset_type, + source="sqlite", + source_path=db_path, + column_mapping=column_mapping, + results=results, + ) diff --git a/report_analyst/core/cache_manager.py b/report_analyst/core/cache_manager.py index caf48b49..7132fa92 100644 --- a/report_analyst/core/cache_manager.py +++ b/report_analyst/core/cache_manager.py @@ -6,9 +6,8 @@ from typing import Any, Dict, List, Optional import numpy as np -from llama_index.core import Document, QueryBundle -from llama_index.core.indices import VectorStoreIndex -from sqlalchemy import text +from llama_index.core import Document +from sqlalchemy import bindparam, text from .database_manager import DatabaseManager from .database_schema import indexes, metadata @@ -17,7 +16,7 @@ class CacheManager: - def __init__(self, db_path: str = None, database_url: str = None): + def __init__(self, db_path: str | None = None, database_url: str | None = None): """ Initialize CacheManager. @@ -58,6 +57,8 @@ def __init__(self, db_path: str = None, database_url: str = None): # In-memory vector store for current document self.vector_store = None self.current_file_path = None + self.current_chunk_size = None + self.current_chunk_overlap = None def init_db(self): """Initialize the database schema using SQLAlchemy""" @@ -73,19 +74,24 @@ def init_db(self): for index_sql in indexes: try: conn.execute(text(index_sql)) - except Exception as e: + except Exception as e: # noqa: BLE001 # Index might already exist, which is fine logger.debug(f"Index creation (may already exist): {e}") conn.commit() logger.info("Database schema initialized successfully") except Exception as e: - logger.error(f"Error initializing database schema: {str(e)}", exc_info=True) + logger.error(f"Error initializing database schema: {e!s}", exc_info=True) raise def _load_vector_store(self, file_path: str, chunks: List[Dict]) -> None: """Load chunks into an in-memory vector store.""" try: + # Clear the existing vector store if it exists + if self.vector_store is not None: + logger.debug("Clearing existing vector store before reload") + self.vector_store = None + # Convert chunks to Documents documents = [] for chunk in chunks: @@ -102,21 +108,57 @@ def _load_vector_store(self, file_path: str, chunks: List[Dict]) -> None: ) documents.append(doc) - # Create vector store index with pre-computed embeddings + if not documents: + logger.warning( + f"No documents with embeddings found to load into vector store. Total chunks provided: {len(chunks)}" + ) + # Clear vector store if no documents to load + self.vector_store = None + self.current_file_path = file_path + + # Derive chunk parameters from the first chunk if available + if chunks: + self.current_chunk_size = chunks[0].get("chunk_size") + self.current_chunk_overlap = chunks[0].get("chunk_overlap") + else: + self.current_chunk_size = None + self.current_chunk_overlap = None + + logger.warning( + "Vector store cleared - no chunks available for chunk_size=%s, chunk_overlap=%s", + self.current_chunk_size, + self.current_chunk_overlap, + ) + return + + # Create vector store index with pre-computed embeddings. + # MockEmbedding avoids requiring OPENAI_API_KEY when every document + # already carries an embedding (tests and offline cache reload). + from llama_index.core.embeddings import MockEmbedding from llama_index.core.indices.vector_store.base import VectorStoreIndex + first_embedding = documents[0].embedding + embed_dim = len(first_embedding) self.vector_store = VectorStoreIndex.from_documents( documents, + embed_model=MockEmbedding(embed_dim=embed_dim), store_nodes_override=True, # Keep nodes in memory use_async=False, # Synchronous operation since we have embeddings show_progress=True, # Show progress during index creation ) self.current_file_path = file_path + # Store chunk parameters to detect when they change + if chunks: + self.current_chunk_size = chunks[0].get("chunk_size") + self.current_chunk_overlap = chunks[0].get("chunk_overlap") - logger.info(f"Loaded {len(documents)} chunks into vector store for {file_path}") + logger.info( + f"Loaded {len(documents)} chunks into vector store for {file_path} " + f"(chunk_size={self.current_chunk_size}, chunk_overlap={self.current_chunk_overlap})" + ) except Exception as e: - logger.error(f"Error loading vector store: {str(e)}", exc_info=True) + logger.error(f"Error loading vector store: {e!s}", exc_info=True) raise async def get_similar_chunks( @@ -124,16 +166,44 @@ async def get_similar_chunks( query_embedding: np.ndarray, file_path: str, top_k: int = 5, - chunk_size: int = None, - chunk_overlap: int = None, + chunk_size: int | None = None, + chunk_overlap: int | None = None, ) -> List[Dict]: """Get chunks most similar to the query embedding using LlamaIndex vector store.""" try: # Load chunks into vector store if needed - if self.current_file_path != file_path: + # Check if file path OR chunk parameters changed + needs_reload = ( + self.current_file_path != file_path + or self.current_chunk_size != chunk_size + or self.current_chunk_overlap != chunk_overlap + ) + + if needs_reload: + logger.info( + f"Reloading vector store: file_path changed={self.current_file_path != file_path}, " + f"chunk_size changed={self.current_chunk_size != chunk_size}, " + f"chunk_overlap changed={self.current_chunk_overlap != chunk_overlap}" + ) chunks = self.get_document_chunks(file_path, chunk_size, chunk_overlap) + logger.info(f"Retrieved {len(chunks)} chunks for chunk_size={chunk_size}, chunk_overlap={chunk_overlap}") + if not chunks: + logger.warning( + f"No chunks found in database for file_path={file_path}, " + f"chunk_size={chunk_size}, chunk_overlap={chunk_overlap}. " + f"Vector store will be empty. This may cause incorrect similarity search results." + ) self._load_vector_store(file_path, chunks) + # Verify vector store is loaded + if self.vector_store is None: + logger.error( + f"Vector store is None for file_path={file_path}, " + f"chunk_size={chunk_size}, chunk_overlap={chunk_overlap}. " + f"Cannot perform similarity search. Chunks may not exist in database." + ) + return [] + # Get similar nodes using vector store retriever = self.vector_store.as_retriever(similarity_top_k=top_k) @@ -156,6 +226,17 @@ async def get_similar_chunks( node.score if hasattr(node, "score") else node.get_score() if hasattr(node, "get_score") else 0.0 ) + node_chunk_size = node.metadata.get("chunk_size") + node_chunk_overlap = node.metadata.get("chunk_overlap") + + # Verify chunk metadata matches requested parameters + if node_chunk_size != chunk_size or node_chunk_overlap != chunk_overlap: + logger.warning( + f"Chunk metadata mismatch! Requested chunk_size={chunk_size}, chunk_overlap={chunk_overlap}, " + f"but chunk has chunk_size={node_chunk_size}, chunk_overlap={node_chunk_overlap}. " + f"Chunk text preview: {node.text[:100]}..." + ) + chunk = { "id": node.metadata.get("id"), "text": node.text, @@ -165,7 +246,11 @@ async def get_similar_chunks( "similarity_score": similarity_score, # Also store as 'similarity_score' for backward compatibility } chunks.append(chunk) - logger.debug(f"Found chunk with similarity score: {similarity_score:.4f}") + logger.debug( + f"Found chunk (chunk_size={node_chunk_size}, chunk_overlap={node_chunk_overlap}) " + f"with similarity score: {similarity_score:.4f}, " + f"text preview: {node.text[:100]}..." + ) logger.info(f"Retrieved {len(chunks)} similar chunks for {file_path}") if chunks: @@ -176,7 +261,7 @@ async def get_similar_chunks( return chunks except Exception as e: - logger.error(f"Error getting similar chunks: {str(e)}", exc_info=True) + logger.error(f"Error getting similar chunks: {e!s}", exc_info=True) return [] def save_analysis(self, file_path: str, question_id: str, result: Dict, config: Dict): @@ -194,7 +279,7 @@ def save_analysis(self, file_path: str, question_id: str, result: Dict, config: result_obj = conn.execute( text( """ - SELECT id FROM questions + SELECT id FROM questions WHERE question_id = :question_id AND question_set = :question_set """ ), @@ -325,7 +410,7 @@ def save_analysis(self, file_path: str, question_id: str, result: Dict, config: result_obj = conn.execute( text( """ - SELECT id FROM document_chunks + SELECT id FROM document_chunks WHERE file_path = :file_path AND chunk_text = :chunk_text """ ), @@ -389,10 +474,13 @@ def save_analysis(self, file_path: str, question_id: str, result: Dict, config: }, ) logger.info( - f"Saving raw values to DB - similarity_score: {chunk.get('similarity_score')}, llm_score: {chunk.get('llm_score')}, is_evidence: {chunk.get('is_evidence')}" + "Saving raw values to DB - similarity_score: %s, llm_score: %s, is_evidence: %s", + chunk.get("similarity_score"), + chunk.get("llm_score"), + chunk.get("is_evidence"), ) else: - logger.warning(f"Could not find chunk in document_chunks table") + logger.warning("Could not find chunk in document_chunks table") # Save to analysis cache logger.info("Saving to analysis cache") @@ -403,9 +491,19 @@ def save_analysis(self, file_path: str, question_id: str, result: Dict, config: INSERT INTO analysis_cache (file_path, question_id, chunk_size, chunk_overlap, top_k, model, question_set, result, created_at) - VALUES (:file_path, :question_id, :chunk_size, :chunk_overlap, :top_k, - :model, :question_set, :result, :created_at) - ON CONFLICT (file_path, question_id, chunk_size, chunk_overlap, top_k, model, question_set) DO UPDATE + VALUES ( + :file_path, + :question_id, + :chunk_size, + :chunk_overlap, + :top_k, + :model, + :question_set, + :result, + :created_at + ) + ON CONFLICT (file_path, question_id, chunk_size, chunk_overlap, top_k, model, question_set) + DO UPDATE SET result = EXCLUDED.result, created_at = EXCLUDED.created_at """ @@ -429,8 +527,17 @@ def save_analysis(self, file_path: str, question_id: str, result: Dict, config: INSERT OR REPLACE INTO analysis_cache (file_path, question_id, chunk_size, chunk_overlap, top_k, model, question_set, result, created_at) - VALUES (:file_path, :question_id, :chunk_size, :chunk_overlap, :top_k, - :model, :question_set, :result, :created_at) + VALUES ( + :file_path, + :question_id, + :chunk_size, + :chunk_overlap, + :top_k, + :model, + :question_set, + :result, + :created_at + ) """ ), { @@ -449,7 +556,7 @@ def save_analysis(self, file_path: str, question_id: str, result: Dict, config: logger.info("Successfully saved complete analysis") except Exception as e: - logger.error(f"Error saving analysis: {str(e)}", exc_info=True) + logger.error(f"Error saving analysis: {e!s}", exc_info=True) raise def get_analysis(self, file_path: str, config: Dict, question_ids: Optional[List[str]] = None) -> Dict[str, Any]: @@ -477,6 +584,7 @@ def get_analysis(self, file_path: str, config: Dict, question_ids: Optional[List "tcfd": "tcfd", "s4m": "s4m", "lucia": "lucia", + "climretrieve": "climretr", # Map climretrieve to climretr shortcut } db_question_set = question_set_mapping.get(config["question_set"], config["question_set"]) @@ -522,33 +630,33 @@ def get_analysis(self, file_path: str, config: Dict, question_ids: Optional[List # Now get the chunk information for each question if results: - # Build IN clause for question IDs - qid_placeholders = ",".join(f":qid_{i}" for i in range(len(results))) - chunk_query = f""" - SELECT - ac.question_id, - dc.chunk_text, - dc.metadata as chunk_metadata, - cr.chunk_order, - cr.similarity_score, - cr.llm_score, - cr.is_evidence, - cr.evidence_order, - cr.metadata as relevance_metadata - FROM analysis_cache ac - JOIN questions q ON q.question_id = ac.question_id - JOIN question_analysis qa ON qa.question_id = q.id AND qa.file_path = ac.file_path - JOIN chunk_relevance cr ON cr.question_analysis_id = qa.id - JOIN document_chunks dc ON cr.document_chunk_id = dc.id - WHERE ac.file_path = :file_path - AND ac.chunk_size = :chunk_size - AND ac.chunk_overlap = :chunk_overlap - AND ac.top_k = :top_k - AND ac.model = :model - AND ac.question_set = :question_set - AND ac.question_id IN ({qid_placeholders}) - ORDER BY ac.question_id, cr.chunk_order - """ + chunk_query = text( + """ + SELECT + ac.question_id, + dc.chunk_text, + dc.metadata as chunk_metadata, + cr.chunk_order, + cr.similarity_score, + cr.llm_score, + cr.is_evidence, + cr.evidence_order, + cr.metadata as relevance_metadata + FROM analysis_cache ac + JOIN questions q ON q.question_id = ac.question_id + JOIN question_analysis qa ON qa.question_id = q.id AND qa.file_path = ac.file_path + JOIN chunk_relevance cr ON cr.question_analysis_id = qa.id + JOIN document_chunks dc ON cr.document_chunk_id = dc.id + WHERE ac.file_path = :file_path + AND ac.chunk_size = :chunk_size + AND ac.chunk_overlap = :chunk_overlap + AND ac.top_k = :top_k + AND ac.model = :model + AND ac.question_set = :question_set + AND ac.question_id IN :question_ids + ORDER BY ac.question_id, cr.chunk_order + """ + ).bindparams(bindparam("question_ids", expanding=True)) chunk_params = { "file_path": str(file_path), @@ -558,11 +666,10 @@ def get_analysis(self, file_path: str, config: Dict, question_ids: Optional[List "model": config["model"], "question_set": db_question_set, } - for i, qid in enumerate(results.keys()): - chunk_params[f"qid_{i}"] = qid + chunk_params["question_ids"] = list(results.keys()) - logger.info(f"Executing chunk query with params: {list(chunk_params.keys())}") - chunk_result = conn.execute(text(chunk_query), chunk_params) + logger.info("Executing chunk query with params: %s", list(chunk_params.keys())) + chunk_result = conn.execute(chunk_query, chunk_params) chunk_rows = chunk_result.fetchall() logger.info(f"Retrieved {len(chunk_rows)} chunk rows") @@ -580,23 +687,28 @@ def get_analysis(self, file_path: str, config: Dict, question_ids: Optional[List "relevance_metadata": json.loads(row[8]) if row[8] else {}, } logger.info( - f"Raw DB values for chunk - similarity_score: {row[4]}, llm_score: {row[5]}, is_evidence: {row[6]}" + "Raw DB values for chunk - similarity_score: %s, llm_score: %s, is_evidence: %s", + row[4], + row[5], + row[6], ) results[question_id]["chunks"].append(chunk_info) # Sort chunks by their order for question_id in results: results[question_id]["chunks"].sort(key=lambda x: x["chunk_order"]) - logger.info(f"Question {question_id}: {len(results[question_id]['chunks'])} chunks") + logger.info("Question %s: %s chunks", question_id, len(results[question_id]["chunks"])) if results[question_id]["chunks"]: logger.info( - f" Similarity range: {min(c['similarity_score'] for c in results[question_id]['chunks']):.4f} - {max(c['similarity_score'] for c in results[question_id]['chunks']):.4f}" + " Similarity range: %.4f - %.4f", + min(c["similarity_score"] for c in results[question_id]["chunks"]), + max(c["similarity_score"] for c in results[question_id]["chunks"]), ) return results except Exception as e: - logger.error(f"Error retrieving analysis: {str(e)}", exc_info=True) + logger.error(f"Error retrieving analysis: {e!s}", exc_info=True) raise def save_vectors(self, file_path: str, chunks: List[Dict[str, Any]]) -> None: @@ -638,8 +750,8 @@ def save_vectors(self, file_path: str, chunks: List[Dict[str, Any]]) -> None: "created_at": datetime.now().isoformat(), } ) - except Exception as e: - logger.warning(f"Error preparing chunk {i} for storage: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Error preparing chunk {i} for storage: {e!s}") continue if chunk_data: @@ -653,8 +765,15 @@ def save_vectors(self, file_path: str, chunks: List[Dict[str, Any]]) -> None: INSERT INTO document_chunks (file_path, chunk_text, chunk_size, chunk_overlap, embedding, metadata, created_at) - VALUES (:file_path, :chunk_text, :chunk_size, :chunk_overlap, - :embedding, :metadata, :created_at) + VALUES ( + :file_path, + :chunk_text, + :chunk_size, + :chunk_overlap, + :embedding, + :metadata, + :created_at + ) ON CONFLICT (file_path, chunk_text, chunk_size, chunk_overlap) DO UPDATE SET embedding = EXCLUDED.embedding, metadata = EXCLUDED.metadata, @@ -672,8 +791,15 @@ def save_vectors(self, file_path: str, chunks: List[Dict[str, Any]]) -> None: INSERT OR REPLACE INTO document_chunks (file_path, chunk_text, chunk_size, chunk_overlap, embedding, metadata, created_at) - VALUES (:file_path, :chunk_text, :chunk_size, :chunk_overlap, - :embedding, :metadata, :created_at) + VALUES ( + :file_path, + :chunk_text, + :chunk_size, + :chunk_overlap, + :embedding, + :metadata, + :created_at + ) """ ), chunk_row, @@ -692,7 +818,7 @@ def save_vectors(self, file_path: str, chunks: List[Dict[str, Any]]) -> None: logger.warning("No valid chunks to save") except Exception as e: - logger.error(f"Error saving vectors: {str(e)}", exc_info=True) + logger.error(f"Error saving vectors: {e!s}", exc_info=True) raise def get_vectors(self, file_path: str) -> List[Dict[str, Any]]: @@ -726,7 +852,7 @@ def get_vectors(self, file_path: str) -> List[Dict[str, Any]]: else: # Fallback to default shape if not stored embedding = np.frombuffer(row[1], dtype=np.float32) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning(f"Error reconstructing embedding: {e}") embedding = None @@ -745,7 +871,7 @@ def get_vectors(self, file_path: str) -> List[Dict[str, Any]]: logger.info(f"Retrieved {len(chunks)} vectors for {file_path}") return chunks except Exception as e: - logger.error(f"Error retrieving vectors: {str(e)}", exc_info=True) + logger.error(f"Error retrieving vectors: {e!s}", exc_info=True) return [] def clear_cache(self, file_path: Optional[str] = None): @@ -767,10 +893,13 @@ def clear_cache(self, file_path: Optional[str] = None): conn.execute(text("DELETE FROM document_chunks")) logger.info("Cleared all cache") except Exception as e: - logger.error(f"Error clearing cache: {str(e)}", exc_info=True) + logger.error(f"Error clearing cache: {e!s}", exc_info=True) def list_analysis_keys(self) -> List[Dict[str, str]]: - """List distinct (file_path, question_set) pairs that have stored analysis. Used for UI dropdowns driven by stored data.""" + """List distinct (file_path, question_set) pairs that have stored analysis. + + Used for UI dropdowns driven by stored data. + """ try: with self.db_manager.get_connection() as conn: result_obj = conn.execute( @@ -788,7 +917,7 @@ def list_analysis_keys(self) -> List[Dict[str, str]]: logger.error(f"Error listing analysis keys: {e}", exc_info=True) return [] - def check_cache_status(self, file_path: str = None): + def check_cache_status(self, file_path: str | None = None): """Debug method to check cache contents""" try: with self.db_manager.get_connection() as conn: @@ -823,7 +952,7 @@ def check_cache_status(self, file_path: str = None): return rows except Exception as e: - logger.error(f"Error checking cache status: {str(e)}", exc_info=True) + logger.error(f"Error checking cache status: {e!s}", exc_info=True) return [] def get_all_answers_by_question_set(self, question_set: str) -> Dict[str, Any]: @@ -901,7 +1030,7 @@ def save_document_chunks(self, file_path: str, chunks: List[Dict], chunk_size: i with self.db_manager.get_connection() as conn: for i, chunk in enumerate(chunks): - logger.debug(f"Processing chunk {i+1}/{len(chunks)}") + logger.debug(f"Processing chunk {i + 1}/{len(chunks)}") if "embedding" not in chunk or chunk["embedding"] is None: logger.warning(f"Skipping chunk {i} - no valid embedding") @@ -919,7 +1048,15 @@ def save_document_chunks(self, file_path: str, chunks: List[Dict], chunk_size: i """ INSERT INTO document_chunks (file_path, chunk_text, chunk_size, chunk_overlap, embedding, metadata, created_at) - VALUES (:file_path, :chunk_text, :chunk_size, :chunk_overlap, :embedding, :metadata, :created_at) + VALUES ( + :file_path, + :chunk_text, + :chunk_size, + :chunk_overlap, + :embedding, + :metadata, + :created_at + ) ON CONFLICT (file_path, chunk_text, chunk_size, chunk_overlap) DO UPDATE SET embedding = EXCLUDED.embedding, metadata = EXCLUDED.metadata, @@ -942,7 +1079,15 @@ def save_document_chunks(self, file_path: str, chunks: List[Dict], chunk_size: i """ INSERT OR REPLACE INTO document_chunks (file_path, chunk_text, chunk_size, chunk_overlap, embedding, metadata, created_at) - VALUES (:file_path, :chunk_text, :chunk_size, :chunk_overlap, :embedding, :metadata, :created_at) + VALUES ( + :file_path, + :chunk_text, + :chunk_size, + :chunk_overlap, + :embedding, + :metadata, + :created_at + ) """ ), { @@ -962,7 +1107,7 @@ def save_document_chunks(self, file_path: str, chunks: List[Dict], chunk_size: i result_obj = conn.execute( text( """ - SELECT COUNT(*) FROM document_chunks + SELECT COUNT(*) FROM document_chunks WHERE file_path = :file_path AND chunk_size = :chunk_size AND chunk_overlap = :chunk_overlap """ ), @@ -976,10 +1121,12 @@ def save_document_chunks(self, file_path: str, chunks: List[Dict], chunk_size: i logger.info(f"Verification: Found {count} chunks in database for {file_path}") except Exception as e: - logger.error(f"Error saving document chunks: {str(e)}", exc_info=True) + logger.error(f"Error saving document chunks: {e!s}", exc_info=True) raise - def get_document_chunks(self, file_path: str, chunk_size: int = None, chunk_overlap: int = None) -> List[Dict]: + def get_document_chunks( + self, file_path: str, chunk_size: int | None = None, chunk_overlap: int | None = None + ) -> List[Dict]: """ Get document chunks from cache with improved logging. """ @@ -1053,10 +1200,12 @@ def get_document_chunks(self, file_path: str, chunk_size: int = None, chunk_over return chunks except Exception as e: - logger.error(f"Error getting document chunks: {str(e)}", exc_info=True) + logger.error(f"Error getting document chunks: {e!s}", exc_info=True) return [] - def get_chunks_without_embeddings(self, file_path: str, chunk_size: int = None, chunk_overlap: int = None) -> List[Dict]: + def get_chunks_without_embeddings( + self, file_path: str, chunk_size: int | None = None, chunk_overlap: int | None = None + ) -> List[Dict]: """Get chunks without embeddings (where embedding IS NULL)""" try: logger.info(f"Retrieving chunks without embeddings for {file_path}") @@ -1108,7 +1257,7 @@ def get_chunks_without_embeddings(self, file_path: str, chunk_size: int = None, return chunks except Exception as e: - logger.error(f"Error getting chunks without embeddings: {str(e)}", exc_info=True) + logger.error(f"Error getting chunks without embeddings: {e!s}", exc_info=True) return [] def has_chunk_scoring(self, file_path: str, config: Dict) -> bool: @@ -1135,6 +1284,6 @@ def has_chunk_scoring(self, file_path: str, config: Dict) -> bool: count = result_obj.fetchone()[0] return count > 0 - except Exception as e: - logger.error(f"Error checking chunk scoring: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error checking chunk scoring: {e!s}") return False diff --git a/report_analyst/core/llm_models.py b/report_analyst/core/llm_models.py new file mode 100644 index 00000000..3c9442db --- /dev/null +++ b/report_analyst/core/llm_models.py @@ -0,0 +1,115 @@ +"""LLM model lists for Streamlit UI and Report Analyst API. + +Environment overrides (comma-separated model IDs): + +- ``OPENAI_MODELS`` — OpenAI dropdown/API entries (replaces default OpenAI list) +- ``GEMINI_MODELS`` — Gemini entries when ``GOOGLE_API_KEY`` is set +- ``LLM_MODELS`` — full combined list (overrides OpenAI + Gemini merge) + +Default selection: ``OPENAI_API_MODEL`` or ``DEFAULT_MODEL``, else first entry in +the active list. +""" + +from __future__ import annotations + +import os +import re + +DEFAULT_OPENAI_MODELS: tuple[str, ...] = ( + "gpt-5.4-mini", + "gpt-5.4", + "gpt-4o-mini", + "gpt-4o", + "gpt-4-turbo", + "gpt-3.5-turbo", +) + +DEFAULT_GEMINI_MODELS: tuple[str, ...] = ( + "gemini-3.5-flash", + "gemini-3.1-pro-preview", + "gemini-3.1-flash-lite", + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-1.5-flash", + "gemini-1.5-pro", +) + + +def _parse_csv_models(env_value: str | None) -> list[str]: + if not env_value or not env_value.strip(): + return [] + return [part.strip() for part in env_value.split(",") if part.strip()] + + +def _dedupe_preserve_order(models: list[str]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for model in models: + if model not in seen: + seen.add(model) + out.append(model) + return out + + +def get_openai_models() -> list[str]: + override = _parse_csv_models(os.getenv("OPENAI_MODELS")) + if override: + return override + return list(DEFAULT_OPENAI_MODELS) + + +def get_gemini_models() -> list[str]: + override = _parse_csv_models(os.getenv("GEMINI_MODELS")) + if override: + return override + return list(DEFAULT_GEMINI_MODELS) + + +def get_llm_models(*, include_gemini: bool | None = None) -> list[str]: + combined = _parse_csv_models(os.getenv("LLM_MODELS")) + if combined: + return _dedupe_preserve_order(combined) + + models = list(get_openai_models()) + if include_gemini is None: + include_gemini = bool(os.getenv("GOOGLE_API_KEY")) + if include_gemini: + models.extend(get_gemini_models()) + return _dedupe_preserve_order(models) + + +def get_default_llm_model() -> str: + preferred = (os.getenv("OPENAI_API_MODEL") or os.getenv("DEFAULT_MODEL") or "").strip() + models = get_llm_models() + if preferred: + if preferred in models: + return preferred + if preferred.startswith("gpt-") or preferred.startswith("gemini-"): + return preferred + return models[0] if models else "gpt-4o-mini" + + +def llm_model_index(models: list[str], preferred: str) -> int: + try: + return models.index(preferred) + except ValueError: + return 0 + + +def model_display_name(model_id: str) -> str: + """Human-readable label for API / UI.""" + if model_id.startswith("gpt-"): + body = model_id.removeprefix("gpt-") + parts = body.split("-") + titled = " ".join(p.upper() if p.isdigit() or p in {"mini", "nano", "pro"} else p.title() for p in parts) + return f"GPT-{titled}" + if model_id.startswith("gemini-"): + body = model_id.removeprefix("gemini-") + parts = re.split(r"[-.]", body) + titled = " ".join(p.upper() if p.isdigit() else p.title() for p in parts if p) + return f"Gemini {titled}" + return model_id + + +def get_models_for_api(*, include_gemini: bool | None = None) -> list[dict[str, str]]: + return [{"id": mid, "name": model_display_name(mid)} for mid in get_llm_models(include_gemini=include_gemini)] diff --git a/report_analyst/core/storage/benchmark_store.py b/report_analyst/core/storage/benchmark_store.py new file mode 100644 index 00000000..f1ce7883 --- /dev/null +++ b/report_analyst/core/storage/benchmark_store.py @@ -0,0 +1,402 @@ +import hashlib +import json +import logging +import sqlite3 +from datetime import datetime +from typing import Dict, List, Optional + +from ...models.benchmark import ( + BenchmarkDataset, + BenchmarkDatasetContent, + BenchmarkEvaluation, + EvaluationMetrics, + HumanAnnotation, + RetrievalConfig, +) + +logger = logging.getLogger(__name__) + + +class BenchmarkStore: + """Handle database operations for benchmarking data""" + + def __init__(self, db_path: str): + self.db_path = db_path + self.init_db() + + def init_db(self): + """Initialize the database schema for benchmarking tables""" + with sqlite3.connect(self.db_path) as conn: + # Create benchmark_datasets table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS benchmark_datasets ( + dataset_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + version TEXT, + question_set TEXT, + file_path TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + + # Create ground_truth_chunks table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS ground_truth_chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dataset_id TEXT NOT NULL, + question_id TEXT NOT NULL, + chunk_id TEXT NOT NULL, + relevance_score REAL DEFAULT 0.0, + is_evidence INTEGER DEFAULT 0, + evidence_order INTEGER, + annotation_notes TEXT, + FOREIGN KEY (dataset_id) REFERENCES benchmark_datasets(dataset_id), + UNIQUE(dataset_id, question_id, chunk_id) + ) + """ + ) + + # Create benchmark_evaluations table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS benchmark_evaluations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dataset_id TEXT NOT NULL, + evaluation_name TEXT NOT NULL, + config_hash TEXT, + retrieval_config TEXT, + evaluation_metrics TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (dataset_id) REFERENCES benchmark_datasets(dataset_id) + ) + """ + ) + + # Create human_annotations table + conn.execute( + """ + CREATE TABLE IF NOT EXISTS human_annotations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + evaluation_id INTEGER NOT NULL, + question_id TEXT NOT NULL, + chunk_id TEXT NOT NULL, + human_relevance_score REAL, + human_is_evidence INTEGER, + human_evidence_order INTEGER, + annotation_notes TEXT, + annotator_id TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (evaluation_id) REFERENCES benchmark_evaluations(id) + ) + """ + ) + + # Create indices for better performance + conn.execute("CREATE INDEX IF NOT EXISTS idx_ground_truth_dataset ON ground_truth_chunks(dataset_id)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_ground_truth_question ON ground_truth_chunks(question_id)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_evaluations_dataset ON benchmark_evaluations(dataset_id)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_annotations_evaluation ON human_annotations(evaluation_id)") + + conn.commit() + logger.info("Benchmark database schema initialized") + + def save_dataset(self, dataset: BenchmarkDatasetContent, file_path: str) -> int: + """Save a benchmark dataset to the database""" + with sqlite3.connect(self.db_path) as conn: + # Insert dataset metadata + cursor = conn.execute( + """ + INSERT OR REPLACE INTO benchmark_datasets + (dataset_id, name, description, version, question_set, file_path, updated_at) + VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + """, + ( + dataset.dataset_id, + dataset.name, + dataset.description, + dataset.version, + dataset.question_set, + file_path, + ), + ) + + # Clear existing ground truth data for this dataset + conn.execute( + "DELETE FROM ground_truth_chunks WHERE dataset_id = ?", + (dataset.dataset_id,), + ) + + # Insert ground truth chunks + for question in dataset.questions: + for chunk in question.ground_truth_chunks: + conn.execute( + """ + INSERT INTO ground_truth_chunks + (dataset_id, question_id, chunk_id, relevance_score, is_evidence, evidence_order, annotation_notes) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + dataset.dataset_id, + question.question_id, + chunk.chunk_id, + chunk.relevance_score, + chunk.is_evidence, + chunk.evidence_order, + chunk.annotation_notes, + ), + ) + + conn.commit() + logger.info(f"Saved dataset {dataset.dataset_id} with {len(dataset.questions)} questions") + return cursor.lastrowid + + def get_dataset(self, dataset_id: str) -> Optional[BenchmarkDataset]: + """Retrieve a benchmark dataset by ID""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.execute( + """ + SELECT * FROM benchmark_datasets WHERE dataset_id = ? + """, + (dataset_id,), + ) + + row = cursor.fetchone() + if not row: + return None + + return BenchmarkDataset( + dataset_id=row["dataset_id"], + name=row["name"], + description=row["description"] or None, + version=row["version"] or None, + question_set=row["question_set"] or None, + file_path=row["file_path"] or None, + created_at=(datetime.fromisoformat(row["created_at"]) if row["created_at"] else None), + updated_at=(datetime.fromisoformat(row["updated_at"]) if row["updated_at"] else None), + source="database", + ) + + def list_datasets(self) -> List[BenchmarkDataset]: + """List all available benchmark datasets""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.execute( + """ + SELECT * FROM benchmark_datasets ORDER BY created_at DESC + """ + ) + + datasets = [] + for row in cursor.fetchall(): + datasets.append( + BenchmarkDataset( + dataset_id=row["dataset_id"], + name=row["name"], + description=row["description"] or None, + version=row["version"] or None, + question_set=row["question_set"] or None, + source="database", + file_path=row["file_path"], + created_at=(datetime.fromisoformat(row["created_at"]) if row["created_at"] else None), + updated_at=(datetime.fromisoformat(row["updated_at"]) if row["updated_at"] else None), + ) + ) + + return datasets + + def get_ground_truth(self, dataset_id: str) -> Dict[str, Dict[str, float]]: + """Get ground truth data for a dataset""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.execute( + """ + SELECT question_id, chunk_id, relevance_score + FROM ground_truth_chunks + WHERE dataset_id = ? + """, + (dataset_id,), + ) + + ground_truth = {} + for row in cursor.fetchall(): + question_id = row["question_id"] + if question_id not in ground_truth: + ground_truth[question_id] = {} + ground_truth[question_id][row["chunk_id"]] = row["relevance_score"] + + return ground_truth + + def save_evaluation(self, evaluation: BenchmarkEvaluation) -> int: + """Save an evaluation result to the database""" + config_json = evaluation.retrieval_config.model_dump_json() + config_hash = hashlib.sha256(config_json.encode()).hexdigest()[:16] + metrics_json = evaluation.evaluation_metrics.model_dump_json() + + with sqlite3.connect(self.db_path) as conn: + cursor = conn.execute( + """ + INSERT INTO benchmark_evaluations + (dataset_id, evaluation_name, config_hash, retrieval_config, evaluation_metrics) + VALUES (?, ?, ?, ?, ?) + """, + ( + evaluation.dataset_id, + evaluation.evaluation_name, + config_hash, + config_json, + metrics_json, + ), + ) + + conn.commit() + evaluation_id = cursor.lastrowid + logger.info(f"Saved evaluation {evaluation.evaluation_name} with ID {evaluation_id}") + return evaluation_id + + def get_evaluation(self, evaluation_id: int) -> Optional[BenchmarkEvaluation]: + """Retrieve an evaluation by ID""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.execute( + """ + SELECT * FROM benchmark_evaluations WHERE id = ? + """, + (evaluation_id,), + ) + + row = cursor.fetchone() + if not row: + return None + + config_dict = json.loads(row["retrieval_config"]) + metrics_dict = json.loads(row["evaluation_metrics"]) + + return BenchmarkEvaluation( + id=row["id"], + dataset_id=row["dataset_id"], + evaluation_name=row["evaluation_name"], + config_hash=row["config_hash"], + retrieval_config=RetrievalConfig(**config_dict), + evaluation_metrics=EvaluationMetrics(**metrics_dict), + created_at=(datetime.fromisoformat(row["created_at"]) if row["created_at"] else None), + ) + + def list_evaluations(self, dataset_id: Optional[str] = None) -> List[BenchmarkEvaluation]: + """List evaluations, optionally filtered by dataset""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + + if dataset_id: + cursor = conn.execute( + """ + SELECT * FROM benchmark_evaluations + WHERE dataset_id = ? + ORDER BY created_at DESC + """, + (dataset_id,), + ) + else: + cursor = conn.execute( + """ + SELECT * FROM benchmark_evaluations + ORDER BY created_at DESC + """ + ) + + evaluations = [] + for row in cursor.fetchall(): + config_dict = json.loads(row["retrieval_config"]) + metrics_dict = json.loads(row["evaluation_metrics"]) + + evaluations.append( + BenchmarkEvaluation( + id=row["id"], + dataset_id=row["dataset_id"], + evaluation_name=row["evaluation_name"], + config_hash=row["config_hash"], + retrieval_config=RetrievalConfig(**config_dict), + evaluation_metrics=EvaluationMetrics(**metrics_dict), + created_at=(datetime.fromisoformat(row["created_at"]) if row["created_at"] else None), + ) + ) + + return evaluations + + def save_annotation(self, annotation: HumanAnnotation) -> int: + """Save a human annotation""" + with sqlite3.connect(self.db_path) as conn: + cursor = conn.execute( + """ + INSERT OR REPLACE INTO human_annotations + (evaluation_id, question_id, chunk_id, human_relevance_score, + human_is_evidence, human_evidence_order, annotation_notes, annotator_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + annotation.evaluation_id, + annotation.question_id, + annotation.chunk_id, + annotation.human_relevance_score, + annotation.human_is_evidence, + annotation.human_evidence_order, + annotation.annotation_notes, + annotation.annotator_id, + ), + ) + + conn.commit() + return cursor.lastrowid + + def get_annotations(self, evaluation_id: int) -> List[HumanAnnotation]: + """Get all annotations for an evaluation""" + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + cursor = conn.execute( + """ + SELECT * FROM human_annotations + WHERE evaluation_id = ? + ORDER BY created_at DESC + """, + (evaluation_id,), + ) + + annotations = [] + for row in cursor.fetchall(): + annotations.append( + HumanAnnotation( + id=row["id"], + evaluation_id=row["evaluation_id"], + question_id=row["question_id"], + chunk_id=row["chunk_id"], + human_relevance_score=row["human_relevance_score"], + human_is_evidence=row["human_is_evidence"], + human_evidence_order=row["human_evidence_order"], + annotation_notes=row["annotation_notes"], + annotator_id=row["annotator_id"], + created_at=(datetime.fromisoformat(row["created_at"]) if row["created_at"] else None), + ) + ) + + return annotations + + def delete_dataset(self, dataset_id: str) -> bool: + """Delete a dataset and all related data""" + with sqlite3.connect(self.db_path) as conn: + # Delete in order due to foreign key constraints + conn.execute( + "DELETE FROM human_annotations WHERE evaluation_id IN " + "(SELECT id FROM benchmark_evaluations WHERE dataset_id = ?)", + (dataset_id,), + ) + conn.execute("DELETE FROM benchmark_evaluations WHERE dataset_id = ?", (dataset_id,)) + conn.execute("DELETE FROM ground_truth_chunks WHERE dataset_id = ?", (dataset_id,)) + cursor = conn.execute("DELETE FROM benchmark_datasets WHERE dataset_id = ?", (dataset_id,)) + + conn.commit() + return cursor.rowcount > 0 diff --git a/report_analyst/data/cache/analysis.db b/report_analyst/data/cache/analysis.db deleted file mode 100644 index 9ab7fa19..00000000 Binary files a/report_analyst/data/cache/analysis.db and /dev/null differ diff --git a/report_analyst/gradio_app.py b/report_analyst/gradio_app.py deleted file mode 100644 index 705f1aa9..00000000 --- a/report_analyst/gradio_app.py +++ /dev/null @@ -1,247 +0,0 @@ -import json -import logging -import os -import shutil -import tempfile -import uuid -from pathlib import Path -from typing import AsyncGenerator, Dict, List - -import gradio as gr -from core.analyzer import DocumentAnalyzer -from dotenv import load_dotenv - -# Load environment variables -load_dotenv() - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[ - logging.StreamHandler(), # Log to console - logging.FileHandler("app.log"), # Log to file - ], -) -logger = logging.getLogger(__name__) - - -class DocumentService: - def __init__(self): - self.analyzer = DocumentAnalyzer() - # Get valid question IDs from the loaded questions - self.valid_question_ids = list(range(1, len(self.analyzer.questions["TCFD Analysis"]["questions"]) + 1)) - logger.info(f"Initialized with {len(self.valid_question_ids)} valid question IDs") - - def validate_question_ids(self, question_ids: List[int]) -> List[int]: - """Validate and filter question IDs""" - if not question_ids: - raise ValueError("No questions selected") - - valid_ids = [qid for qid in question_ids if qid in self.valid_question_ids] - if not valid_ids: - raise ValueError("No valid questions selected") - - logger.info(f"Validated question IDs: {valid_ids}") - return valid_ids - - async def process_document(self, file_path: str, question_ids: List[int] = None) -> AsyncGenerator[Dict, None]: - """Process uploaded document and stream analysis results""" - if not file_path: - yield {"error": "No file uploaded"} - return - - try: - # Validate question IDs only if they are provided - if question_ids is not None: - question_ids = self.validate_question_ids(question_ids) - logger.info(f"Processing questions: {question_ids}") - else: - # If no questions specified, use all valid IDs - question_ids = self.valid_question_ids - logger.info("No questions specified, using all questions") - - temp_file = Path(tempfile.gettempdir()) / f"temp_{uuid.uuid4()}.pdf" - try: - shutil.copy2(file_path, temp_file) - async for result in self.analyzer.process_document(str(temp_file), question_ids): - logger.info(f"Processing section: {result.get('section', 'unknown')}") - yield result - finally: - if temp_file.exists(): - temp_file.unlink() - except Exception as e: - logger.error(f"Error processing document: {str(e)}") - yield {"error": f"Failed to process document: {str(e)}"} - - -def create_app(): - service = DocumentService() - progress_tracker = gr.Progress() - - with gr.Blocks( - title="TCFD Report Analyzer", - theme=gr.themes.Soft(), - css=""" - .question-result { - margin-bottom: 20px; - padding: 15px; - border: 1px solid #ddd; - border-radius: 8px; - } - .question-result h3 { - margin-top: 0; - } - """, - ) as app: - gr.Markdown("# TCFD Report Analyzer") - - with gr.Tabs() as tabs: - # Analysis Tab - with gr.Tab("Analysis"): - gr.Markdown("Upload a sustainability report for detailed TCFD analysis") - - with gr.Row(): - file_input = gr.File(label="Upload PDF Report", file_types=[".pdf"], type="filepath") - analyze_btn = gr.Button("Start Analysis", variant="primary") - - with gr.Row(): - progress = gr.Markdown("Upload a report to begin analysis") - - # Results Section - with gr.Column(visible=False) as results_container: - with gr.Row(): - gr.Markdown("## Analysis Results") - - # Accordion for each question - questions_accordion = gr.Accordion("Questions", open=False) - - # DataFrame for summary view - summary_df = gr.Dataframe( - headers=["Question", "Score", "Answer Summary"], - label="Analysis Summary", - visible=False, - ) - - # Questions Tab - with gr.Tab("Questions"): - gr.Markdown("## TCFD Questions") - gr.Markdown("Select which questions to include in the analysis") - - with gr.Row(): - select_all = gr.Button("Select All", variant="secondary") - clear_all = gr.Button("Clear All", variant="secondary") - - questions = service.analyzer.questions["TCFD Analysis"]["questions"] - checkboxes = [] - - for i, question in enumerate(questions, 1): - checkbox = gr.Checkbox( - label=f"Q{i}: {question}", - value=True, # Default to selected - interactive=True, - ) - checkboxes.append(checkbox) - - async def process_analysis(file, *selected_questions): - results_container.visible = False - summary_data = [] - - if not file: - yield "Please upload a file first", [], {} - return - - try: - selected_ids = [i + 1 for i, selected in enumerate(selected_questions) if selected] - if not selected_ids: - yield "Please select at least one question", [], {} - return - - questions_html = "" - - async for result in service.process_document(file, selected_ids): - if "error" in result: - yield f"Error: {result['error']}", [], {} - return - - if "status" in result: - progress_tracker(0, desc=result["status"]) - continue - - try: - analysis = json.loads(result["result"]) - - # Create accordion item for this question - score_html = f'
Score: {analysis.get("score", "N/A")}
' - - question_html = f""" -
-

Question {result['question_number']}

- {score_html} -

Q: {result['question']}

-

A: {analysis.get('answer', 'No answer provided')}

- """ - - if analysis.get("evidence"): - question_html += "

Evidence:

    " - question_html += "".join([f"
  • {e}
    • " for e in analysis["evidence"]]) - question_html += "
" - - if analysis.get("gaps"): - question_html += "

Gaps:

    " - question_html += "".join([f"
  • {g}
    • " for g in analysis["gaps"]]) - question_html += "
" - - question_html += "
" - questions_html += question_html - - # Add to summary data - summary_data.append( - [ - f"Q{result['question_number']}", - analysis.get("score", "N/A"), - analysis.get("answer", "No answer")[:100] + "...", - ] - ) - - # Update UI - results_container.visible = True - yield questions_html, summary_data, {"visible": True} - - except json.JSONDecodeError: - logger.error("Failed to parse analysis result") - continue - - progress_tracker(1.0, desc="Analysis complete!") - - except Exception as e: - logger.error(f"Error in analysis: {str(e)}", exc_info=True) - yield f"Error: {str(e)}", [], {} - - def select_all_questions(): - """Select all questions""" - logger.info("Selecting all questions") - return [True] * len(checkboxes) - - def clear_all_questions(): - """Clear all question selections""" - logger.info("Clearing all questions") - return [False] * len(checkboxes) - - # Connect components - analyze_btn.click( - fn=process_analysis, - inputs=[file_input] + checkboxes, - outputs=[questions_accordion, summary_df, results_container], - ) - - select_all.click(fn=select_all_questions, inputs=[], outputs=checkboxes) - - clear_all.click(fn=clear_all_questions, inputs=[], outputs=checkboxes) - - return app - - -if __name__ == "__main__": - app = create_app() - app.launch(server_name="0.0.0.0", server_port=7860, share=True, debug=True) diff --git a/report_analyst/models/benchmark.py b/report_analyst/models/benchmark.py new file mode 100644 index 00000000..8f759e2b --- /dev/null +++ b/report_analyst/models/benchmark.py @@ -0,0 +1,235 @@ +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + + +# Define enums first +class DatasetType(str, Enum): + """Types of benchmark datasets""" + + INFORMATION_RETRIEVAL = "information_retrieval" # IR: Compare retrieved chunks + INFORMATION_EXTRACTION = "information_extraction" # IE: Compare analysis/answers + + +class FlexibleDatasetRow(BaseModel): + """ + Flexible row model that can handle different column names and structures. + All fields are stored in a flexible data dictionary. + """ + + data: Dict[str, Any] = Field(..., description="Flexible data dictionary containing all row fields") + + def get(self, key: str, default: Any = None) -> Any: + """Get value by key (case-insensitive)""" + key_lower = key.lower() + for k, v in self.data.items(): + if k.lower() == key_lower: + return v + return default + + def get_query_id(self) -> Optional[str]: + """Get query/question ID using common column name variations""" + return self.get("query_id") or self.get("question_id") or self.get("qid") + + def get_chunk_id(self) -> Optional[str]: + """Get chunk ID using common column name variations""" + return self.get("chunk_id") or self.get("chunk") or self.get("cid") + + def get_score(self) -> Optional[float]: + """Get score using common column name variations""" + score = ( + self.get("score") or self.get("relevance_score") or self.get("confidence_score") or self.get("similarity_score") + ) + if score is not None: + try: + return float(score) + except (ValueError, TypeError): + return None + return None + + def get_position(self) -> Optional[int]: + """Get position/rank using common column name variations""" + pos = self.get("position") or self.get("rank") or self.get("order") + if pos is not None: + try: + return int(pos) + except (ValueError, TypeError): + return None + return None + + def get_answer(self) -> Optional[str]: + """Get answer/analysis text for IE datasets""" + return self.get("answer") or self.get("analysis") or self.get("text") or self.get("response") + + def get_category(self) -> Optional[str]: + """Get category/classification""" + return self.get("category") or self.get("class") or self.get("label") or self.get("type") + + +class BenchmarkDataset(BaseModel): + """ + Unified flexible benchmark dataset model that handles different formats, column names, and dataset types. + Supports both Information Retrieval (IR) and Information Extraction (IE) datasets. + Can be used for metadata-only (database storage) or with full results (CSV/SQLite loaded). + """ + + # Core identifiers + dataset_id: str = Field(..., description="Unique identifier for this dataset") + name: str = Field(..., description="Name of the dataset") + description: Optional[str] = Field(None, description="Description of the dataset") + + # Metadata fields + version: Optional[str] = Field(None, description="Version of the dataset") + question_set: Optional[str] = Field(None, description="Question set this dataset is for") + file_path: Optional[str] = Field(None, description="Path to the dataset file") + created_at: Optional[datetime] = Field(None, description="Creation timestamp") + updated_at: Optional[datetime] = Field(None, description="Last update timestamp") + + # Dataset type and source (for loaded datasets) + dataset_type: Optional[DatasetType] = Field(None, description="Type of dataset: IR or IE (None for metadata-only)") + source: Optional[str] = Field(None, description="Source: 'csv', 'sqlite', 'internal', 'database'") + source_path: Optional[str] = Field(None, description="Path to source file/database") + + # Data and mapping (for loaded datasets) + column_mapping: Dict[str, str] = Field( + default_factory=dict, + description="Mapping of standard names to actual column names", + ) + results: List[FlexibleDatasetRow] = Field( + default_factory=list, + description="List of dataset rows (empty for metadata-only)", + ) + + def get_results_by_query(self, query_id: str) -> List[FlexibleDatasetRow]: + """Get all results for a specific query/question""" + return [r for r in self.results if r.get_query_id() == query_id] + + def get_unique_queries(self) -> List[str]: + """Get list of unique query/question IDs""" + query_ids = [r.get_query_id() for r in self.results if r.get_query_id()] + return sorted(list(set(query_ids))) + + def get_unique_reports(self) -> List[str]: + """Get list of unique report/document IDs""" + report_ids = [r.get("report_id") or r.get("document_id") or r.get("doc_id") for r in self.results] + report_ids = [r for r in report_ids if r] + return sorted(list(set(report_ids))) + + def is_metadata_only(self) -> bool: + """Check if this is a metadata-only dataset (no results loaded)""" + return len(self.results) == 0 + + def has_results(self) -> bool: + """Check if this dataset has results loaded""" + return len(self.results) > 0 + + +class GroundTruthChunk(BaseModel): + chunk_id: str = Field(..., description="Identifier for the chunk") + relevance_score: float = Field(..., ge=0.0, le=1.0, description="Ground truth relevance score") + is_evidence: bool = Field(..., description="Whether this chunk is evidence") + evidence_order: Optional[int] = Field(None, description="Order of evidence if applicable") + annotation_notes: Optional[str] = Field(None, description="Notes about this annotation") + text: Optional[str] = Field(None, description="Optional chunk text for error analysis") + metadata: Optional[Dict[str, Any]] = Field(default=None, description="Optional chunk metadata") + + +class BenchmarkQuestion(BaseModel): + question_id: str = Field(..., description="Question identifier") + question_text: str = Field(..., description="The actual question text") + ground_truth_chunks: List[GroundTruthChunk] = Field(..., description="Ground truth chunks for this question") + + +class BenchmarkDatasetContent(BaseModel): + dataset_id: str + name: str + description: str + version: str + question_set: str + created_at: str + questions: List[BenchmarkQuestion] + + +class RetrievalConfig(BaseModel): + chunk_size: int = Field(default=1000, description="Size of text chunks") + chunk_overlap: int = Field(default=200, description="Overlap between chunks") + top_k: int = Field(default=5, description="Number of chunks to retrieve") + use_llm_scoring: bool = Field(default=False, description="Whether to use LLM scoring") + embedding_model: str = Field(default="default", description="Embedding model to use") + similarity_threshold: float = Field(default=0.0, description="Minimum similarity threshold") + llm_model: Optional[str] = Field(None, description="LLM model for scoring") + + +class EvaluationMetrics(BaseModel): + precision_at_k: Dict[int, float] = Field(default_factory=dict, description="Precision at different K values") + recall_at_k: Dict[int, float] = Field(default_factory=dict, description="Recall at different K values") + f1_at_k: Dict[int, float] = Field(default_factory=dict, description="F1 score at different K values") + mean_reciprocal_rank: float = Field(default=0.0, description="Mean reciprocal rank") + mean_average_precision: float = Field(default=0.0, description="Mean average precision") + ndcg_at_k: Dict[int, float] = Field(default_factory=dict, description="NDCG at different K values") + + +class BenchmarkEvaluation(BaseModel): + id: Optional[int] = None + dataset_id: str = Field(..., description="Dataset used for evaluation") + evaluation_name: str = Field(..., description="Name of this evaluation") + config_hash: str = Field(..., description="Hash of the retrieval configuration") + retrieval_config: RetrievalConfig = Field(..., description="Configuration used for retrieval") + evaluation_metrics: EvaluationMetrics = Field(..., description="Computed evaluation metrics") + created_at: Optional[datetime] = None + + +class HumanAnnotation(BaseModel): + id: Optional[int] = None + evaluation_id: int = Field(..., description="Evaluation this annotation belongs to") + question_id: str = Field(..., description="Question being annotated") + chunk_id: str = Field(..., description="Chunk being annotated") + human_relevance_score: float = Field(..., ge=0.0, le=1.0, description="Human-assigned relevance score") + human_is_evidence: bool = Field(..., description="Human judgment on evidence") + human_evidence_order: Optional[int] = Field(None, description="Human-assigned evidence order") + annotation_notes: Optional[str] = Field(None, description="Notes from the annotator") + annotator_id: str = Field(..., description="ID of the person making the annotation") + created_at: Optional[datetime] = None + + +# Legacy models for backward compatibility + + +class RetrievalResultRow(BaseModel): + """Single row in a retrieval results dataset (CSV format) - Legacy model""" + + query_id: str = Field(..., description="Question/query identifier") + report_id: Optional[str] = Field(None, description="Report/document identifier") + chunk_id: str = Field(..., description="Chunk identifier") + chunk_text: Optional[str] = Field(None, description="Chunk text content") + position: int = Field(..., ge=1, description="Position/rank in retrieval results (1-indexed)") + score: float = Field(..., description="Retrieval score (similarity, relevance, etc.)") + similarity_score: Optional[float] = Field(None, description="Vector similarity score") + llm_score: Optional[float] = Field(None, description="LLM-based relevance score") + metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Additional metadata") + + +class RetrievalResultsDataset(BaseModel): + """Dataset containing retrieval results (from CSV or SQLite) - Legacy model""" + + dataset_id: str = Field(..., description="Unique identifier for this dataset") + name: str = Field(..., description="Name of the dataset") + description: Optional[str] = Field(None, description="Description of the dataset") + source: str = Field(..., description="Source: 'csv', 'sqlite', 'internal'") + source_path: Optional[str] = Field(None, description="Path to source file/database") + results: List[RetrievalResultRow] = Field(..., description="List of retrieval results") + created_at: Optional[datetime] = Field(None, description="Creation timestamp") + + def get_results_by_query(self, query_id: str) -> List[RetrievalResultRow]: + """Get all retrieval results for a specific query""" + return [r for r in self.results if r.query_id == query_id] + + def get_unique_queries(self) -> List[str]: + """Get list of unique query IDs""" + return sorted(list(set(r.query_id for r in self.results))) + + def get_unique_reports(self) -> List[str]: + """Get list of unique report IDs""" + return sorted(list(set(r.report_id for r in self.results if r.report_id))) diff --git a/report_analyst/questionsets/climretrieve_questions.yaml b/report_analyst/questionsets/climretrieve_questions.yaml new file mode 100644 index 00000000..417aad79 --- /dev/null +++ b/report_analyst/questionsets/climretrieve_questions.yaml @@ -0,0 +1,793 @@ +name: ClimRetrieve +shortcut: climretr +description: Comprehensive sustainability labeling framework for evaluating climate transformation initiatives and carbon + footprint reporting +questions: +- id: climretr_1 + text: Does the company provide definitions for climate change adaptation? + guidelines: '' +- id: climretr_2 + text: Has the company identified any synergies between its climate change adaptation goals and other business goals? + guidelines: '' +- id: climretr_3 + text: Does the company report the climate change scenarios used to test the resilience of its business strategy? + guidelines: '' +- id: climretr_4 + text: Does the company seek to adjust its business model to better provide climate change adaptation products and services? + guidelines: '' +- id: climretr_5 + text: Does the company have any engagements with industry peers in relation to climate change? + guidelines: '' +- id: climretr_6 + text: Does the company have a specific process in place to identify risks arising from climate change? + guidelines: '' +- id: climretr_7 + text: Does the company report the methodology used to identify the dependencies and impact of its business activities on + the environment? + guidelines: '' +- id: climretr_8 + text: Does the company refer to any third party scenarios when identifying climate-related risks or opportunities (e.g. + IPCC trajectories, NGFS scenarios, etc.)? + guidelines: '' +- id: climretr_9 + text: Does the company encourage downstream partners to carry out climate-related risk assessments? + guidelines: '' +- id: climretr_10 + text: Does the company report how adjustments to its business operations will allow it to adapt to climate change? + guidelines: '' +- id: climretr_11 + text: Do the environmental/sustainability targets set by the company reference external climate change adaptation goals/targets? + guidelines: '' +- id: climretr_12 + text: Do the environmental/sustainability targets set by the company align with external climate change adaptation goals/targets? + guidelines: '' +- id: climretr_13 + text: Does the company report short-term actions taken or planned to reduce its waste generation? + guidelines: '' +- id: climretr_14 + text: Does the company report a plan to engage with downstream partners on water consumption or water pollution? + guidelines: '' +- id: climretr_15 + text: Does the company identify any impacts of its business activities on the environment? + guidelines: '' +- id: climretr_16 + text: Does the company have a strategy on waste management? + guidelines: '' +- id: climretr_17_ir + text: We search for details on how a company's efforts to adapt to climate change are integrated with and support its broader + business objectives. This includes examining how initiatives aimed at increasing climate resilience also contribute to + the company's long-term sustainability, competitive advantage, stakeholder engagement, and overall economic, social, and + environmental impact. Information of interest encompasses strategies that align climate action with enhancing client success, + community strengthening, and opportunity advancement. We look for evidence of how environmental, social, and governance + (ESG) goals, particularly those related to climate action, are woven into the fabric of the company's mission, operational + practices, and corporate culture. This also involves understanding how these efforts are communicated and reported, reflecting + a commitment to transparency and stakeholder engagement. Additionally, we seek insights into how the company's climate + change adaptation measures are perceived as adding value to its brand, customer satisfaction, and investor appeal, thereby + driving business growth and shareholder returns. + guidelines: '' +- id: climretr_18_ir + text: We search for details on the specific climate change scenarios a company uses to evaluate the resilience of its business + strategy. This includes identifying the types of scenarios (e.g., IEA Net Zero Emissions Scenario, high emission scenarios) + and the time frames considered (e.g., 2030, 2050). Information on how these scenarios help the company to identify, assess, + and prioritize climate-related risks and opportunities is crucial. This encompasses both physical risks, such as increased + temperatures and sea levels affecting operations, and transition risks, including policy changes or shifts in market preferences. + Additionally, we look for how these analyses are integrated into the company’s risk management processes, such as through + an Enterprise Risk Register, and how they inform strategic planning and decision-making. Details on mitigation strategies + and opportunities identified through this process, including financial impacts and actions taken to adapt or capitalize + on climate-related trends, are also relevant. This approach ensures a comprehensive understanding of how prepared a company + is to navigate the challenges and opportunities presented by climate change across various industries. + guidelines: '' +- id: climretr_19_ir + text: We search for details on how a company is innovating and transforming its operations, products, or services to address + the challenges posed by climate change. This includes investments in sustainability initiatives, such as funds dedicated + to environmental projects or the adoption of renewable energy sources, which demonstrate a commitment to reducing carbon + footprints and enhancing energy efficiency. Information on the development or implementation of technologies that minimize + energy loss, improve resource use efficiency, or contribute to the sustainability of operations is also pertinent. Additionally, + we look for evidence of strategic planning, such as the adoption of sustainability guides or frameworks that prioritize + eco-friendly practices in new projects or renovations. The establishment of programs that encourage sustainable practices + within specific departments or units, like laboratories, indicates a holistic approach to embedding climate resilience + across the company's operations. Overall, we seek to understand how a company is pivoting towards sustainability to mitigate + the impacts of climate change and ensure long-term resilience. + guidelines: '' +- id: climretr_20_ir + text: We search for details on how a company collaborates with other entities within its industry to address climate change. + This includes partnerships with stakeholders such as suppliers, non-profit organizations, industry groups, governments, + and customers aimed at promoting sustainability and achieving net-zero emissions. Information on joint initiatives with + other corporations, participation in industry-wide commitments to reduce greenhouse gas emissions, and engagement in collective + actions to tackle environmental challenges, like deforestation and responsible sourcing, is pertinent. Additionally, membership + in industry-specific sustainability organizations or initiatives that work towards creating a more sustainable sector, + whether it's in cocoa, coffee, tea, or any other, demonstrates a company's commitment to collaborative efforts in combating + climate change. The involvement in partnerships that focus on innovation for scalable solutions and the integration of + sustainable practices across the supply chain are also key indicators of a company's engagement with industry peers in + relation to climate change adaptation and resilience. + guidelines: '' +- id: climretr_21_ir + text: We search for details on how a company identifies and addresses risks related to climate change, focusing on both + the transition to a low-carbon economy and the physical impacts of climate change. This involves looking into whether + the company has integrated Environmental, Social, and Governance (ESG) considerations into its risk management strategies, + thereby reducing its environmental footprint, enhancing diversity, and creating long-term value. We also seek information + on specific tools or programs the company uses to assess climate-related risks, such as providing or utilizing localized + climate data to inform investment decisions and improve community resilience. Additionally, we look for evidence of the + company's commitment to making infrastructure more resilient against climate change, which could include investments in + sustainable technologies or practices. This inquiry spans across all industries, aiming to understand how companies are + preparing for and adapting to the challenges posed by climate change in a holistic manner. + guidelines: '' +- id: climretr_22_ir + text: We search for details on how a company identifies and evaluates the ways in which its operations depend on and impact + the environment. This involves understanding the methodologies or frameworks the company uses to assess its environmental + footprint, including both direct and indirect effects on ecosystems, biodiversity, and climate. Information of interest + includes the company's approach to sourcing materials sustainably, efforts to conserve natural resources, and initiatives + aimed at enhancing ecological resilience. For instance, a company might detail its strategies for engaging with suppliers + on sustainable sourcing practices or describe projects it supports that contribute to the restoration and protection of + critical habitats. Additionally, we look for evidence of how these environmental considerations are integrated into the + company's business model, such as investing a portion of profits into conservation and climate adaptation projects. This + information helps gauge the company's commitment to mitigating its environmental impact and adapting to climate change, + showcasing a proactive approach to sustainability and resilience. + guidelines: '' +- id: climretr_23_ir + text: We search for details on whether a company incorporates external, authoritative climate projections and scenarios + into their strategic planning and risk management processes. This involves looking for references to established climate + models and pathways, such as those provided by the Intergovernmental Panel on Climate Change (IPCC) or the Network for + Greening the Financial System (NGFS), which offer a range of future climate conditions based on varying levels of greenhouse + gas emissions. Specifically, we're interested in how these scenarios are used to evaluate potential financial, operational, + and strategic impacts under different climate futures, including both risks and opportunities. This could involve assessing + the implications of transition risks as economies shift towards lower carbon pathways, or the physical risks posed by + changes in climate patterns and extreme weather events. Information that indicates a company is applying these scenarios + to understand direct impacts on their business operations, investment strategies, or risk management practices is particularly + relevant. + guidelines: '' +- id: climretr_24_ir + text: We search for details on how a company motivates and supports its downstream partners, including suppliers and distributors, + in identifying and managing climate-related risks. This involves looking into whether the company has established frameworks + or initiatives that encourage these partners to evaluate their vulnerability to climate change and to adopt measures that + enhance their resilience. Specifically, we're interested in information about supplier engagement strategies, educational + programs aimed at increasing climate risk awareness, and the provision of tools or infrastructure that facilitate the + assessment and reporting of climate impacts. This could encompass efforts to reduce Scope 3 emissions through sustainable + practices across the supply chain, such as promoting energy transition, advocating for deforestation-free agricultural + practices, and encouraging the production and distribution of energy-efficient products and sustainable packaging. The + goal is to understand how the company collaborates with its network to mitigate climate risks and contribute to a more + sustainable and resilient future. + guidelines: '' +- id: climretr_25_ir + text: We search for details on how a company plans and implements strategies to modify its operations in response to the + changing climate. This includes identifying and mitigating risks posed by climate change to their business model, such + as water scarcity, increased frequency of extreme weather events, and the transition to a low-carbon economy. Information + on sustainable resource management, like water conservation and efficient use, and efforts to reduce carbon footprint + through innovative solutions are pertinent. Additionally, strategies to ensure the sustainability of supply chains, such + as sourcing materials from low-deforestation risk areas and enhancing sustainable packaging practices, are relevant. The + adaptation measures should also cover infrastructure resilience, including designing facilities to withstand extreme weather + and implementing effective crisis management for continuity in essential services. Overall, we look for comprehensive + approaches that a company takes to adapt its operations, safeguard assets, and ensure business continuity in the face + of climate change challenges, applicable across various industries. + guidelines: '' +- id: climretr_26_ir + text: We search for details on how a company's environmental or sustainability goals align with broader, external climate + change adaptation and resilience frameworks or targets. This includes commitments to international agreements like the + Paris Climate Agreement, participation in industry-specific alliances such as the Net-Zero Banking Alliance, and adherence + to pathways aimed at limiting global warming, such as the 1.5°C pathway. Information of interest encompasses the setting + of specific, measurable targets for reducing emissions across various sectors of operation, strategies for transitioning + to net-zero emissions by a certain year, and policies against financing projects that contradict global climate goals. + Additionally, we look for evidence of risk assessments conducted in light of climate change scenarios, such as those provided + by the Intergovernmental Panel on Climate Change, and how these assessments inform operational decisions. This inquiry + aims to gauge the depth of a company's commitment to climate change adaptation and resilience, beyond mere compliance, + towards a proactive and strategic alignment with global efforts to mitigate climate change impacts. + guidelines: '' +- id: climretr_27_ir + text: We search for details on how a company's environmental or sustainability goals are designed to support broader efforts + to adapt to and mitigate the impacts of climate change. This involves examining whether these goals are in harmony with + recognized external standards, initiatives, or guidelines that are widely accepted as effective in addressing climate + change challenges. Specifically, we look for information indicating the company's commitment to reducing greenhouse gas + emissions across all scopes, including direct operations and the wider value chain. This includes assessing whether the + company has set science-based targets, adheres to international standards for achieving net-zero emissions, and engages + in practices that contribute to the restoration and protection of natural ecosystems. Additionally, we evaluate the company's + efforts in supplier engagement to ensure that its entire supply chain is moving towards sustainability. The alignment + with external goals and targets, such as those set by the Science Based Targets initiative, ISO guidelines, or recommendations + from research and environmental organizations, is crucial. This alignment demonstrates a company's proactive stance in + contributing to global efforts to combat climate change and its commitment to sustainability. + guidelines: '' +- id: climretr_28_ir + text: We search for details on the specific measures a company is taking or planning to implement to minimize waste production + in the short term. This encompasses initiatives to enhance the sustainability of products through design innovations, + efforts to increase the recyclability and reuse of materials, and strategies to reduce food and plastic waste. Information + on partnerships with suppliers, customers, and technology firms to promote circular economy practices is also pertinent. + Additionally, we look for data on the tangible outcomes of these actions, such as the quantity of waste diverted from + landfills, reductions in the use of virgin plastics, and increases in the use of recycled materials. The company's commitment + to global sustainability goals, such as zero hunger and responsible consumption, through specific targets like achieving + zero food waste to landfill, is another critical aspect. Lastly, educational and community engagement initiatives aimed + at reducing waste generation across the value chain are of interest. + guidelines: '' +- id: climretr_29_ir + text: We search for details on how a company's operations and processes contribute to environmental degradation or climate + change, focusing on both direct and indirect impacts. This includes quantifiable metrics such as greenhouse gas (GHG) + emissions, which are critical for understanding a company's carbon footprint and its efforts towards reduction. Additionally, + we look into the company's practices that might have broader environmental consequences, such as waste generation, water + usage, and pollution. The information sought also encompasses the company's acknowledgment of the negative repercussions + of its business model on the environment, such as the impact of product returns or the lifecycle environmental cost of + its services and products. This analysis helps in assessing the company's awareness and management of its environmental + impact, which is crucial for adaptation and resilience strategies in the face of climate change. This approach is applicable + across all industries, as every sector has unique environmental challenges and opportunities for improvement. + guidelines: '' +- id: climretr_30_ir + text: We search for details on how a company approaches the reduction and management of waste within its operations and + supply chain, aiming to enhance its resilience and adaptability to environmental challenges. This includes strategies + for minimizing the use of single-use plastics and diverting waste from landfills, which are critical for reducing environmental + impact and promoting sustainability. Information of interest encompasses specific targets and timelines for eliminating + single-use plastics and achieving zero waste to landfill, reflecting a commitment to significant environmental goals. + Additionally, we look for evidence of investments in innovative solutions and technologies that facilitate waste reduction, + as well as initiatives to improve operational and fleet efficiency by reducing waste generation. The development of roadmaps + or action plans that detail steps towards these goals, including efforts to engage with suppliers and partners in reducing + packaging and increasing the use of recycled materials, is also pertinent. These strategies demonstrate a company's proactive + approach to waste management, contributing to its overall sustainability and resilience in the face of climate change. + guidelines: '' +- id: climretr_31_simple_ir + text: |- + We search for details on how the company conceptualizes and defines climate change adaptation within its operations and strategic planning. This involves understanding the company's recognition of climate change impacts and its strategies to adjust and respond to these changes to minimize harm or exploit beneficial opportunities. Specifically, we look for: + + 1. The company's definition of climate change adaptation in the context of its business operations. + 2. Examples of how the company identifies vulnerabilities and risks associated with climate change. + 3. Descriptions of strategies or measures the company has implemented or plans to implement to adapt to changing climate conditions. + 4. Any frameworks or standards the company uses to guide its adaptation efforts. + 5. How the company measures or evaluates the effectiveness of its adaptation strategies. + 6. Information on stakeholder engagement in developing adaptation strategies. + + This information helps assess the company's resilience and preparedness in the face of climate change, indicating its long-term sustainability and risk management approach. + guidelines: '' +- id: climretr_32_simple_ir + text: |- + We search for details on how the company's efforts to adapt to climate change align with and potentially enhance its broader business objectives. This involves understanding the concept of synergies, which refers to situations where combined actions produce a greater outcome than the sum of their individual effects. Specifically, in the context of climate change adaptation and resilience, this means looking for ways in which the company's strategies to mitigate climate risks also contribute to its financial health, operational efficiency, market position, or sustainability targets in other areas such as water use, waste reduction, and social responsibility. + + Examples of information we are looking for include: + - Initiatives that reduce carbon footprint and lower operational costs. + - Investments in renewable energy that secure long-term energy supply and open up new business opportunities. + - Projects that enhance ecosystem resilience, which also serve as community engagement or corporate social responsibility efforts. + - Adaptation measures that improve infrastructure resilience, potentially reducing insurance costs and increasing investor confidence. + guidelines: '' +- id: climretr_33_simple_ir + text: |- + We search for details on the specific climate change scenarios that a company has considered to evaluate how resilient its business strategy is to various potential future climates. This involves understanding the meaning behind climate change scenarios, which are projections based on different assumptions about future greenhouse gas emissions, technological advancements, and policy changes. These scenarios help in assessing the vulnerability and adaptability of a company's operations, supply chains, and market demand under different future conditions. + + Examples of information we look for include: + - The types of climate change scenarios considered (e.g., RCP 2.6, RCP 8.5, SSPs). + - How these scenarios impact the company's critical operations and strategic planning. + - Adaptation measures the company plans to implement in response to these scenarios. + - Any resilience-building strategies to mitigate potential negative impacts. + - The process of integrating scenario analysis into business strategy and risk management practices. + - Evidence of scenario planning in long-term business sustainability goals. + guidelines: '' +- id: climretr_34_simple_ir + text: |- + We search for details on whether the company is actively modifying its business strategy to offer products and services that help in adapting to climate change. This involves understanding the concept of climate change adaptation, which refers to actions taken to manage the impacts of climate change by reducing vulnerability and increasing resilience. Resilience here means the ability of a system, community, or economy to withstand and recover from the effects of climate change. + + Examples of information we look for include: + + 1. Development of new products or services designed to mitigate the impacts of climate change, such as drought-resistant crops or flood defense systems. + 2. Changes in the company's operations or supply chain to reduce risks associated with climate change, such as relocating facilities away from flood-prone areas. + 3. Investments in research and development focused on climate change resilience solutions. + 4. Partnerships with governments or NGOs to support community-based adaptation initiatives. + 5. Training programs for employees and local communities on climate change adaptation strategies. + guidelines: '' +- id: climretr_35_simple_ir + text: |- + We search for details on the company's involvement with other organizations within its industry to collectively address climate change. This includes any partnerships, collaborations, or joint initiatives aimed at reducing greenhouse gas emissions, enhancing sustainability practices, or developing adaptive strategies to mitigate climate change impacts. Specifically, we look for: + + 1. Membership in industry-wide sustainability consortia or alliances focused on climate change. + 2. Participation in roundtable discussions or forums with industry peers to share best practices and challenges related to climate adaptation and resilience. + 3. Joint research projects or innovation labs with other companies to develop new technologies or methods for reducing carbon footprints. + 4. Co-authored position papers or policy advocacy efforts aimed at promoting climate-friendly regulations and standards within the industry. + 5. Collaborative supply chain initiatives to ensure environmental sustainability and resilience against climate-related disruptions. + + These examples illustrate the company's commitment to not only addressing climate change individually but also contributing to sector-wide efforts for a more sustainable and resilient future. + guidelines: '' +- id: climretr_36_simple_ir + text: |- + We search for details on the company's methodologies and strategies for recognizing potential threats linked to climate change. This involves understanding how the company anticipates, evaluates, and plans to manage risks related to extreme weather events, rising sea levels, changing agricultural patterns, and regulatory shifts aimed at reducing carbon emissions. Specifically, we look for: + + 1. **Risk Identification Processes**: How the company systematically identifies climate-related risks, including both direct impacts on operations and indirect impacts through supply chains or market changes. + 2. **Assessment Methods**: The criteria or tools used to evaluate the severity and likelihood of identified risks. + 3. **Adaptation Strategies**: Plans or actions the company has in place to adapt to identified risks, ensuring business continuity and resilience. + 4. **Stakeholder Engagement**: How the company involves stakeholders in its risk identification process, including local communities, suppliers, and customers. + 5. **Monitoring and Reviewing**: The process for regularly updating risk assessments and adaptation strategies in response to new climate data or events. + guidelines: '' +- id: climretr_37_simple_ir + text: |- + We search for details on the specific approaches and techniques a company employs to understand how its operations rely on and affect the natural environment. This involves looking into whether the company has a systematic process for evaluating its environmental footprint, including both direct impacts (such as emissions and waste) and indirect impacts (such as resource consumption and effects on biodiversity). Additionally, we're interested in how the company identifies its vulnerability to environmental changes, which could affect its supply chains, operational viability, and overall sustainability. Examples of information we seek include: + + - Descriptions of environmental risk assessments and their methodologies. + - Details on the use of environmental impact assessments (EIAs) or life cycle assessments (LCAs). + - Information on tools or frameworks used to gauge dependency on ecosystem services. + - Evidence of stakeholder engagement in identifying environmental impacts and dependencies. + - Any mention of collaboration with environmental scientists or sustainability experts to refine their impact assessment process. + guidelines: '' +- id: climretr_38_simple_ir + text: |- + We search for details on whether the company utilizes external, authoritative climate projections or models to inform their understanding of climate-related risks and opportunities. This involves looking for references to scenarios developed by recognized bodies such as the Intergovernmental Panel on Climate Change (IPCC) or the Network for Greening the Financial System (NGFS). These scenarios provide various future outlooks based on different levels of greenhouse gas emissions and policy responses, helping organizations to plan for a range of possible climate futures. Specifically, we're interested in: + + 1. Mention of any specific IPCC trajectories, such as Representative Concentration Pathways (RCPs) or Shared Socioeconomic Pathways (SSPs), that the company has considered in their risk assessments or strategy development. + 2. Reference to NGFS scenarios, which are designed to help financial institutions understand and manage the risks climate change poses to the economy. + 3. How these external scenarios have been integrated into the company's climate risk management or opportunity identification processes. + 4. Any other third-party climate scenarios or models referenced in the company's sustainability reporting. + + This information helps assess the company's thoroughness in considering a broad range of potential climate futures in their adaptation and resilience planning. + guidelines: '' +- id: climretr_39_simple_ir + text: |- + We search for details on whether the company actively promotes and supports climate-related risk assessments among its downstream partners, such as suppliers, distributors, and retailers. This involves understanding if the company extends its climate change adaptation and resilience strategies beyond its immediate operations to include the entire supply chain. Specifically, we look for: + + 1. **Partnership Initiatives:** Evidence of programs or initiatives aimed at helping downstream partners assess their vulnerability to climate change impacts. + 2. **Guidance and Resources:** Information on whether the company provides tools, resources, or expertise to assist partners in conducting their risk assessments. + 3. **Collaborative Assessments:** Instances where the company has worked together with partners to identify and evaluate climate risks. + 4. **Policy and Requirements:** Details on any policies requiring downstream partners to conduct climate risk assessments as a condition of doing business. + 5. **Support for Implementation:** Examples of how the company supports partners in implementing adaptation and resilience measures based on the outcomes of their risk assessments. + + This inquiry aims to gauge the company's commitment to fostering a resilient value chain that can withstand climate change impacts. + guidelines: '' +- id: climretr_40_simple_ir + text: |- + We search for details on how a company plans to modify its business practices to remain resilient and competitive in the face of climate change. This involves understanding the company's awareness and response to the risks and opportunities presented by climate change. Specifically, we look for: + + 1. **Risk Assessment**: Information on how the company identifies and evaluates climate-related risks to its operations, supply chain, and market demand. + 2. **Adaptation Strategies**: Details on specific strategies the company is implementing to adapt to climate change, such as investing in renewable energy, improving water efficiency, or sourcing materials more sustainably. + 3. **Operational Changes**: Examples of operational adjustments, like altering production processes, relocating facilities prone to climate impacts, or adopting new technologies to reduce carbon footprint. + 4. **Financial Planning**: Insight into how climate change is factored into the company's financial planning and investment decisions, ensuring long-term sustainability and resilience. + 5. **Stakeholder Engagement**: Information on how the company communicates and collaborates with stakeholders (employees, customers, investors, local communities) on climate change adaptation efforts. + + These details help assess the company's preparedness for the impacts of climate change, ensuring it can sustain operations and thrive in a changing environment. + guidelines: '' +- id: climretr_41_simple_ir + text: We search for details on whether the company's environmental or sustainability objectives align with broader, externally + established climate change adaptation goals or targets. This involves understanding if the company has integrated international, + national, or sector-specific benchmarks and targets into their sustainability strategy, particularly those aimed at enhancing + resilience to climate change and promoting adaptation measures. Examples of such external goals include commitments under + the Paris Agreement, the United Nations Sustainable Development Goals (SDGs), especially SDG 13 (Climate Action), or industry-specific + resilience standards. Information sought includes mentions of specific external frameworks or targets the company commits + to, how these commitments are reflected in their operational or strategic plans, and any evidence of progress towards + these externally referenced goals. This analysis helps assess the company's commitment to contributing to global efforts + in combating climate change and enhancing adaptive capacities within their operations and broader community or ecosystem + they impact. + guidelines: '' +- id: climretr_42_simple_ir + text: |- + We search for details on how the company's environmental or sustainability objectives match up with broader, externally established climate change adaptation goals or targets. This involves understanding the company's commitments to reducing its environmental impact, enhancing resilience against climate change effects, and contributing to global efforts like the Paris Agreement or the Sustainable Development Goals (SDGs). Specifically, we look for: + + 1. The company's specific environmental targets, such as reducing greenhouse gas emissions, increasing renewable energy usage, or improving water efficiency. + 2. How these targets compare with international, national, or industry-specific adaptation goals, such as those outlined in the Paris Agreement or by the Intergovernmental Panel on Climate Change (IPCC). + 3. Evidence of strategic planning or investment in adaptation measures, like infrastructure improvements to withstand extreme weather events or shifts in operational practices to conserve biodiversity. + 4. Partnerships or collaborations with external bodies to enhance climate resilience. + 5. Reporting mechanisms and progress tracking towards these adaptation and resilience goals. + + This analysis helps gauge the company's commitment to and alignment with global efforts to combat climate change through adaptation and resilience strategies. + guidelines: '' +- id: climretr_43_simple_ir + text: |- + We search for details on the specific measures a company has implemented or intends to implement in the near future to minimize its waste production. This inquiry delves into the realm of adaptation and resilience by examining how a company adjusts its operations to mitigate environmental impact, thereby contributing to a more sustainable and resilient ecosystem. Adaptation refers to the adjustments in processes, practices, or structures to counteract potential damage or take advantage of opportunities arising from environmental changes. Resilience is the capacity to recover quickly from difficulties; in this context, it's about the company's ability to sustain operations amidst environmental challenges by reducing waste generation. + + Examples of information sought include: + - Initiatives to improve resource efficiency or recycling programs aimed at reducing operational waste. + - Investment in technology or processes that minimize waste production. + - Partnerships with waste management firms to ensure responsible waste disposal. + - Employee training programs on sustainability practices to reduce waste. + - Short-term goals or targets set for waste reduction and progress reports on these objectives. + guidelines: '' +- id: climretr_44_simple_ir + text: |- + We search for details on how the company plans to collaborate with its downstream partners, such as suppliers, distributors, and customers, to address and manage water consumption and water pollution issues. This involves understanding the company's strategies for reducing water usage and minimizing water pollution across its value chain. Specifically, we look for: + + 1. **Partnership Initiatives**: Any programs or projects the company has initiated with its downstream partners to reduce water consumption or mitigate water pollution. + 2. **Guidance and Support**: Information on whether the company provides technical support, resources, or guidelines to help partners implement water-saving practices or pollution control measures. + 3. **Monitoring and Reporting**: Details on how the company and its partners monitor water usage and pollution levels, including any shared reporting mechanisms or platforms. + 4. **Goals and Targets**: Specific targets or objectives set by the company in collaboration with its partners to reduce water consumption and pollution over a defined period. + 5. **Success Stories**: Examples of successful water conservation or pollution reduction initiatives undertaken with downstream partners. + + This information is crucial for assessing the company's commitment to sustainability, particularly in managing water resources responsibly and building resilience against water-related challenges in its supply chain. + guidelines: '' +- id: climretr_45_simple_ir + text: |- + We search for details on how the company's operations influence the environment, focusing on both direct and indirect impacts. This involves understanding the company's carbon footprint, waste generation, water usage, and effects on biodiversity. In the context of adaptation and resilience, we're particularly interested in how these environmental impacts might affect the company's ability to adapt to climate change and maintain resilience against environmental stressors. For example, we look for information on: + + - Emissions of greenhouse gases and efforts to reduce them. + - Strategies for managing water resources sustainably and adapting to water scarcity. + - Measures to reduce waste and promote recycling, contributing to a circular economy. + - Initiatives to protect and restore biodiversity, ensuring ecosystem services that support climate adaptation. + - Assessments of how environmental degradation could affect the company's operations and supply chain, and plans to mitigate these risks. + + This information helps gauge the company's commitment to minimizing its environmental footprint and enhancing its resilience to climate change. + guidelines: '' +- id: climretr_46_simple_ir + text: |- + We search for details on the company's approach to handling, reducing, and repurposing waste as part of its operations and product lifecycle. This involves understanding the company's policies, practices, and goals related to waste management, which is crucial for assessing its adaptation and resilience to climate change. Effective waste management strategies can significantly reduce environmental impact, contribute to the circular economy, and enhance the company's sustainability profile. + + Examples of information sought include: + + 1. **Waste Reduction Initiatives:** Efforts to minimize waste production at the source. + 2. **Recycling Programs:** Systems in place for recycling materials used in operations and products. + 3. **Waste Repurposing:** Strategies for reusing waste materials in new or existing products. + 4. **Partnerships:** Collaborations with external organizations for waste management and recycling. + 5. **Compliance and Reporting:** Adherence to local and international waste management regulations and transparent reporting of waste generation and handling statistics. + 6. **Innovation in Waste Management:** Adoption of new technologies or methods for waste treatment and reduction. + 7. **Employee and Community Engagement:** Programs to engage employees and the community in waste reduction and recycling efforts. + + These details help evaluate the company's commitment to sustainability and its capability to adapt to environmental challenges through responsible waste management. + guidelines: '' +- id: climretr_47_ir_three + text: We search for details on how a company conceptualizes and integrates the notions of climate change adaptation into + their operations and strategic planning. This involves looking for explicit definitions or descriptions provided by the + company that explain what they understand by climate change adaptation and resilience. This could include their participation + in initiatives or alliances that focus on innovative solutions for adapting to climate change impacts and enhancing resilience, + such as developing new technologies or practices in digital finance that contribute to these goals. Additionally, we seek + information on how these concepts are applied within the company's sustainability strategies, including any specific projects, + partnerships, or commitments made to address climate adaptation and resilience. The aim is to assess the company's awareness + and proactive approach towards mitigating the adverse effects of climate change through adaptation and resilience-building + measures, which are crucial for sustainable business practices across all industries. + guidelines: '' +- id: climretr_48_ir_three + text: We search for details on how a company integrates its climate change adaptation strategies with its broader business + objectives, ensuring these efforts are not siloed but rather contribute to the company's overall success. This includes + understanding the governance structures in place, such as committees or councils, that oversee the alignment of sustainability + goals with business operations and strategic planning. We look for evidence of how environmental, social, and governance + (ESG) considerations are embedded into company policies, including remuneration and risk management frameworks, to incentivize + and prioritize sustainability efforts among senior leadership and across the organization. Information on how the company + engages with stakeholders to prioritize and assess the impact of its ESG goals is crucial, as is the linkage of sustainability + achievements to compensation structures, demonstrating a commitment to these goals at the highest levels. Additionally, + we examine how the company aligns its sustainability initiatives with global standards or goals, such as the United Nations + Sustainable Development Goals, to ensure its efforts contribute to broader societal and environmental objectives. + guidelines: '' +- id: climretr_49_ir_three + text: We search for details on how a company identifies, assesses, and prepares for the various impacts of climate change + on its operations, supply chain, and overall business strategy. This includes understanding the specific climate-related + scenarios a company considers, such as transitional and physical risks over short, medium, and long-term horizons. Information + on the company's hedging strategies against rising operational costs due to factors like soaring energy prices, as well + as their approach to managing these risks through budgeting and market strategies, is pertinent. Additionally, we look + for evidence of comprehensive climate risk assessments and scenario analyses that guide the company in understanding potential + climate risks and opportunities. This encompasses the integration of climate-related risks into the company's overall + risk management framework, the establishment of emissions-reduction targets, and engagement in initiatives aimed at enhancing + climate adaptation and resilience. The goal is to gauge the company's proactive measures in ensuring its sustainability + and resilience in the face of climate change challenges. + guidelines: '' +- id: climretr_50_ir_three + text: We search for details on how a company is innovating or transforming its operations, products, or services to address + the challenges posed by climate change. This includes the development or integration of technologies that reduce carbon + emissions, such as fuel-efficient transportation or renewable energy sources. We're interested in partnerships that aim + to lower greenhouse gas emissions or initiatives to enhance sustainability in the supply chain, from reducing reliance + on fossil fuels to promoting the use of recycled materials. Efforts to engage with and support communities in adapting + to climate change, through educational programs or by funding research into sustainable practices, are also relevant. + Additionally, strategies to conserve natural resources, like water, and actions to improve environmental and social governance + (ESG) criteria within the company and its suppliers signal a commitment to climate change adaptation. Investments in sustainability, + whether through capital allocation to low-carbon projects or sustainability-linked financial instruments, illustrate the + company's dedication to adjusting its business model for climate resilience. + guidelines: '' +- id: climretr_51_ir_three + text: We search for details on how a company collaborates with others in its industry to address climate change. This includes + sharing best practices, participating in industry associations, and engaging in collective action initiatives. We're interested + in whether the company contributes to or benefits from shared knowledge on sustainability, engages in dialogues with stakeholders + about climate-related issues, and takes part in policy advocacy aligned with climate goals. Information on partnerships + aimed at decarbonizing supply chains, developing sustainable technologies, and setting industry-wide emissions targets + is also relevant. Additionally, we look for evidence of the company working with suppliers and other partners on training + programs to meet climate action goals. The focus is on understanding how the company's collaborative efforts contribute + to broader industry adaptation and resilience to climate change, regardless of the specific sector it operates in. + guidelines: '' +- id: climretr_52_ir_three + text: We search for details on how a company proactively identifies, assesses, and manages risks related to climate change. + This includes the establishment of dedicated teams or offices focusing on environmental affairs and climate risks, and + the integration of climate risk considerations into the company's overall risk management framework. Information on the + role of the Board or specific committees in overseeing climate risk management is pertinent, as well as descriptions of + any frameworks, strategies, or policies developed to address climate risks. Examples of interest also encompass how climate + risks are integrated into investment processes, governance structures, and operational risk frameworks. Additionally, + we look for evidence of scenario analysis, risk identification, and the management practices employed to mitigate adverse + impacts from climate-related events. The company's approach to assessing and managing both the physical and transitional + risks of climate change, including the use of models, stress tests, and the impact on credit ratings and financials, is + also relevant. + guidelines: '' +- id: climretr_53_ir_three + text: We search for details on the specific approaches and frameworks a company employs to understand and evaluate the environmental + dependencies and impacts of its business operations. This includes how they assess the ecological footprint of their activities, + such as the sourcing of raw materials, the operation of facilities, and the delivery of services. Information sought encompasses + initiatives to connect with and support ecosystems and communities through conservation efforts, the allocation of financial + resources to enhance climate resilience and biodiversity, and the adoption of innovative practices like biomimicry to + design solutions that contribute positively to the environment. Additionally, we look for evidence of how these methodologies + contribute to the development of sustainable practices, such as the creation of marine protected areas, coral reef restoration, + or the enhancement of ecosystems around business infrastructures. The goal is to understand not just the impact of the + company's activities but also how they are actively working towards mitigating negative effects and enhancing environmental + resilience. + guidelines: '' +- id: climretr_54_ir_three + text: We search for details on how a company incorporates external climate-related projections and models into their risk + management and opportunity identification processes. This involves looking for references to authoritative sources like + the Intergovernmental Panel on Climate Change (IPCC) assessments, which provide global warming projections and recommendations + for emissions reductions, or the Network for Greening the Financial System (NGFS) scenarios, which offer insights into + the financial risks and opportunities from climate change. Information of interest includes the company's acknowledgment + of global warming impacts as outlined by the IPCC, commitments to international initiatives like the Science-based Targets + initiative or the UNFCCC Race to Zero, alignment with reporting frameworks that consider climate-related financial disclosures + (e.g., TCFD, SASB), and specific strategies to reduce greenhouse gas emissions in line with global climate goals. These + details demonstrate how a company is planning for and adapting to the future climate scenario, enhancing its resilience + against climate-related risks while seizing new opportunities. + guidelines: '' +- id: climretr_55_ir_three + text: We search for details on how a company actively involves its downstream partners, such as suppliers and distributors, + in assessing and mitigating climate-related risks. This involves looking for evidence of strategies and initiatives aimed + at promoting transparency, engagement, and support for these partners to address their environmental impact and sustainability + practices. Specifically, we're interested in whether the company encourages its partners to conduct their own climate + risk assessments, adopt sustainable sourcing practices, and report on their progress in reducing greenhouse gas (GHG) + emissions. Additionally, we seek information on any programs or projects aimed at decarbonization that include supplier + participation, as well as efforts to educate, incentivize, and support suppliers in enhancing their resilience against + climate change. The company's commitment to partnering with suppliers who prioritize high ethical and sustainability standards, + and its efforts to achieve carbon neutrality through collaborative initiatives with its supply chain, are also of interest. + guidelines: '' +- id: climretr_56_ir_three + text: We search for details on how a company plans to modify its operations to withstand and thrive amidst the changing + climate. This encompasses strategies for reducing carbon footprints, such as adopting renewable energy sources, enhancing + operational efficiencies, and investing in advanced, less polluting technologies. Information on the deployment of new, + more efficient product lines or services that contribute to lower CO2 emissions is crucial. We also look for evidence + of innovative practices, like the development and testing of sustainable technologies or materials, and efforts to minimize + waste and resource use across the company's lifecycle. Additionally, details on setting and pursuing ambitious environmental + targets, such as net-zero emissions, reductions in greenhouse gases, energy, water, and waste, are vital. The company's + approach to managing physical risks from climate change, including strategies for business resilience and recovery, alongside + commitments to circular economy practices, are also of interest. These actions demonstrate a company's adaptability to + climate change, showcasing its resilience and commitment to sustainability. + guidelines: '' +- id: climretr_57_ir_three + text: We search for details on how a company's environmental or sustainability goals are aligned with or reference external + climate change adaptation and resilience frameworks, initiatives, or standards. This includes looking for commitments + to internationally recognized targets such as those set by the Science Based Targets initiative (SBTi) for greenhouse + gas (GHG) reduction, alignment with the goals of the Paris Agreement, participation in renewable energy initiatives like + RE100, and adherence to guidelines or standards such as the ISO Net Zero Guidelines or the Task Force on Climate-related + Financial Disclosures (TCFD) methodology. Information indicating the company's efforts to engage with their supply chain + on climate issues, investment in nature-based solutions, and setting specific, science-based emissions reduction targets + for the near and long term are also relevant. Additionally, we look for how these targets and strategies contribute to + broader sustainability goals, such as responsible consumption and production, life below water, and life on land, demonstrating + a holistic approach to environmental responsibility and climate resilience across all operations. + guidelines: '' +- id: climretr_58_ir_three + text: We search for details on how a company's environmental or sustainability goals are designed to meet or exceed established + external climate change adaptation and resilience benchmarks, such as those set by the Paris Agreement, the Net-Zero Banking + Alliance (NZBA), or the Science Based Targets initiative (SBTi). This includes examining commitments to reduce greenhouse + gas (GHG) emissions in alignment with a 1.5°C pathway, efforts to transition to net-zero emissions by a specific year, + and strategies for enhancing climate change resilience. Information on sector-specific targets, especially in carbon-intensive + industries, and the adoption of renewable energy sources to power operations are also pertinent. Additionally, we look + for evidence of engagement with external validation initiatives like SBTi or the Climate Group’s RE100 initiative, which + lend credibility to a company's targets. The exclusion of financing for projects that contradict net-zero ambitions, such + as new coal mines or oil and gas fields, and the setting of science-based targets for emissions reduction across the company's + operations and value chain are also critical details. + guidelines: '' +- id: climretr_59_ir_three + text: We search for details on the specific measures a company is undertaking or planning to implement in the near future + to minimize waste production. This encompasses strategies to enhance material efficiency, such as optimizing the use of + resources in operations, products, and packaging, and prioritizing the sourcing of sustainable materials. Information + on the adoption of circular economy principles, aiming to extend the lifecycle of products through reuse, repair, and + recycling, is also pertinent. Initiatives to phase out single-use plastics and to improve waste management practices, + including the achievement of waste reduction targets and certifications, are relevant. Additionally, efforts to improve + the accuracy of waste data and the expansion of waste disclosure are of interest. These actions demonstrate a company's + commitment to reducing its environmental footprint and contributing to sustainability goals, which are crucial for all + industries aiming to enhance their climate change adaptation and resilience. + guidelines: '' +- id: climretr_60_ir_three + text: We search for details on how a company plans to collaborate with its downstream partners to address water consumption + and pollution issues. This involves understanding the strategies and initiatives a company has in place to not only reduce + its own water footprint but also to influence and work together with its partners, suppliers, and customers to promote + sustainable water use and pollution control. This could include efforts to replenish water sources, innovate new water-saving + technologies, and engage in policy advocacy for water conservation. Additionally, we look for information on how companies + integrate water sustainability into their investment or procurement guidelines, emphasizing the importance of biodiversity, + conservation, and waste management. The goal is to assess the company's commitment to fostering a holistic and collaborative + approach to water sustainability that extends beyond its immediate operations to encompass its entire value chain, thereby + enhancing climate change adaptation and resilience. + guidelines: '' +- id: climretr_61_ir_three + text: We search for details on how a company's operations and activities influence the environment, focusing on both direct + and indirect impacts. This includes emissions of greenhouse gases (GHGs) from company operations and the broader value + chain, such as emissions from purchased goods and services, business travel, and waste generated. Information on the company's + land use, including the extent of land owned, operated, or influenced, and efforts to protect or restore ecosystems, is + also pertinent. Additionally, we look for data on the company's energy consumption, including electricity and other forms + of energy, to understand its carbon footprint. Strategies or initiatives aimed at mitigating environmental impacts, such + as developing biodiversity strategies, collaborating on low-emission transportation solutions, or addressing impacts through + the supply chain and client engagement, are also of interest. This comprehensive view helps assess the company's adaptation + and resilience strategies in the face of climate change challenges. + guidelines: '' +- id: climretr_62_ir_three + text: We search for details on a company's comprehensive approach towards managing and minimizing waste as part of its sustainability + and resilience strategy. This includes initiatives to achieve zero waste through circular economy practices, such as designing + products and packaging to reduce material use, enhancing recyclability, and extending the lifecycle of products through + reuse and repair. Information on specific targets and timelines, like achieving a certain percentage of waste diversion + from landfills by a set year, or the elimination of single-use plastics within the company's operations, is crucial. Additionally, + strategies for increasing the use of recycled materials in products and packaging, and efforts to design out waste from + the beginning, are of interest. The company's commitment to reducing its environmental impact through waste management, + including in areas like paper use and sustainable diets, and how these efforts are integrated across its operations, products, + and supply chain, are also pertinent details. + guidelines: '' +- id: climretr_63_ir_all + text: We search for details on how a company conceptualizes and integrates the notions of climate change adaptation into + their operations and strategic planning. This involves looking for explicit definitions or descriptions provided by the + company that explain what climate change adaptation means to them and how it is relevant to their business model. Specifically, + we are interested in identifying any initiatives, partnerships, or projects the company is involved in that aim to enhance + resilience and adaptability to climate-related challenges. For instance, involvement in alliances or collaborations that + focus on innovative solutions for climate adaptation and resilience, such as developing digital finance tools to support + adaptation efforts, would be pertinent. This information helps assess the company's commitment to and understanding of + the importance of adapting to changing climate conditions, which is crucial across all industries for mitigating risks + and leveraging opportunities related to climate change. + guidelines: '' +- id: climretr_64_ir_all + text: We search for details on how a company integrates its climate change adaptation strategies with its broader business + objectives, ensuring these efforts are not siloed but rather contribute to the company's overall success. This includes + examining the governance structures in place, such as committees or councils, that oversee the alignment of sustainability + goals with business demands. We look into how environmental, social, and governance (ESG) goals are incorporated into + company policies, including remuneration policies, to incentivize sustainable practices among senior leadership and across + the organization. Information on how these sustainability goals are tied to the company's strategic direction, and the + processes for creating and evaluating these goals, is crucial. This involves stakeholder engagement to prioritize ESG + issues and assess risks and opportunities at various levels of the organization. Additionally, we consider how companies + link compensation to sustainability achievements and their commitment to broader global initiatives, such as the United + Nations Sustainable Development Goals, to gauge their external engagement and impact. + guidelines: '' +- id: climretr_65_ir_all + text: We search for details on how companies assess and prepare for the potential impacts of climate change on their business + operations and strategy. This includes identifying whether they conduct climate risk assessments and scenario analyses, + and if so, what specific scenarios they consider (e.g., transitional and physical risks over short, medium, and long terms). + We look for information on how these assessments are integrated into their overall risk management practices, such as + through the application of risk management models or the establishment of emissions-reduction targets based on scientific + data. Additionally, we seek evidence of proactive engagement in initiatives or alliances aimed at enhancing climate adaptation + and resilience, and whether companies are setting clear, science-based targets to mitigate greenhouse gas emissions across + their operations and value chain. The goal is to understand the depth of a company's commitment to adapting its business + strategy to the evolving climate landscape, ensuring long-term resilience and sustainability. + guidelines: '' +- id: climretr_66_ir_all + text: We search for details on how companies are actively modifying their business strategies to incorporate climate change + adaptation products and services. This includes the development and deployment of innovative technologies or processes + that reduce environmental impact, such as the introduction of more fuel-efficient aircraft or the use of sustainable aviation + fuel to lower carbon emissions. It also involves initiatives aimed at enhancing ecosystem resilience and biodiversity, + alongside efforts to reduce water usage and improve water quality in response to climate change challenges. Furthermore, + we look for evidence of companies engaging with their supply chains to promote decarbonization and the use of renewable + energy sources, as well as investments in low-carbon and circular economy projects. The establishment of partnerships + or institutions focused on climate resilience and adaptation strategies, and the allocation of financial resources towards + sustainability-linked projects or bonds, are also pertinent details indicating a company's commitment to adjusting its + business model for climate change adaptation. + guidelines: '' +- id: climretr_67_ir_all + text: We search for details on how a company collaborates with others in its industry to address climate change. This includes + sharing best practices, participating in industry associations, and engaging in collective action initiatives. Information + on partnerships with peers, suppliers, and other stakeholders that aim to reduce emissions, develop sustainable technologies, + or influence policy is relevant. Examples of such collaborations could involve joint development of guidelines for emissions + reporting, advocacy for regulatory changes, participation in industry-wide sustainability training, and projects that + span the entire value chain to decarbonize specific sectors. Additionally, details on how the company aligns its climate + change positions with industry associations and ensures these are in agreement with global climate goals, such as the + Paris Agreement, are pertinent. The involvement in initiatives that support innovation in renewable energy, emissions + reduction, and sustainable practices across the industry, demonstrating a commitment to collective progress towards climate + resilience and adaptation, is also of interest. + guidelines: '' +- id: climretr_68_ir_all + text: We search for details on how a company proactively identifies, assesses, and manages risks related to climate change. + This includes the establishment of specialized teams or offices focused on environmental affairs and climate risk, and + the integration of climate risk considerations into the company's broader risk management frameworks and governance structures. + Information on the development and implementation of policies, standards, and frameworks specifically designed to address + climate risks is crucial. Additionally, we look for evidence of scenario analysis and stress testing to understand potential + impacts of climate change on the company's operations, including both transition risks, such as regulatory changes and + shifts in market preferences, and physical risks, such as extreme weather events. The role of executive oversight, such + as board committees responsible for climate risk oversight, and the incorporation of climate risk into investment processes + and operational risk frameworks, are also key indicators of a company's commitment to identifying and managing climate-related + risks. + guidelines: '' +- id: climretr_69_ir_all + text: We search for details on the specific processes and methodologies a company employs to understand and evaluate how + its operations and business activities interact with and impact the environment. This includes identifying both how the + company depends on natural resources and ecosystems, and how its actions might affect these environmental elements. For + instance, we look for information on initiatives that connect business practices with environmental conservation efforts, + such as platforms that aid in sustainable sourcing or projects aimed at enhancing ecosystem health around business sites. + Additionally, we seek examples of how a company contributes to climate resilience and conservation, through financial + or operational support for projects like Marine Protected Areas or coral reef restoration. The use of innovative approaches, + like biomimicry, to minimize negative impacts on the environment and to promote regeneration and sustainability in business + expansions or operations is also pertinent. These details collectively help in understanding a company's commitment and + approach to environmental sustainability and resilience. + guidelines: '' +- id: climretr_70_ir_all + text: We search for details on how a company incorporates external climate projections and models into their risk management + and opportunity identification processes. This involves looking for references to authoritative sources like the Intergovernmental + Panel on Climate Change (IPCC) assessments, which provide global warming projections and recommendations for emissions + reductions, or the Network for Greening the Financial System (NGFS) scenarios, which offer insights into how financial + systems can evolve in response to climate change. Information that indicates a company is aligning its strategies with + these or similar third-party climate scenarios—such as setting net-zero targets in line with global efforts to limit warming, + adopting governance and reporting standards that consider climate-related financial disclosures, or designing emissions + reduction strategies based on specific global warming potential scenarios—demonstrates their engagement with established + climate science and policy frameworks. This approach helps in assessing the company's preparedness for and resilience + to climate-related challenges and opportunities. + guidelines: '' +- id: climretr_71_ir_all + text: We search for details on how a company actively involves its downstream partners, such as suppliers and distributors, + in identifying, assessing, and mitigating climate-related risks. This involves looking for evidence of the company's efforts + to promote transparency in environmental progress, support for sustainability initiatives within the supply chain, and + encouragement of sustainable sourcing practices. Specifically, we're interested in whether the company asks its partners + to conduct climate risk assessments, report greenhouse gas (GHG) emissions, and adopt decarbonization strategies. Additionally, + we seek information on any educational or financial support provided to partners to enhance their resilience against climate + change impacts. This could include partnerships with suppliers committed to high ethical and environmental standards, + initiatives to build transparent supply chains, and efforts to reduce environmental impacts collaboratively. The goal + is to understand how the company extends its climate change adaptation and resilience strategies beyond its immediate + operations to its broader network of partners, contributing to industry-wide sustainability practices. + guidelines: '' +- id: climretr_72_ir_all + text: We search for details on how a company plans to modify its operations to withstand and thrive amidst the changing + climate. This includes strategies for reducing greenhouse gas emissions, such as adopting renewable energy sources, enhancing + operational efficiency, and investing in advanced technologies that lower carbon footprints. Information on the deployment + of new, more efficient product designs or materials that contribute to emissions reduction is pertinent. Additionally, + we look for evidence of efforts to minimize resource use, including water and energy, and strategies for waste reduction, + such as circular practices that extend product lifecycles and reduce landfill waste. Details on setting and achieving + ambitious sustainability targets, like net-zero emissions and significant reductions in water use and solid waste, are + crucial. Furthermore, we examine how companies assess and mitigate the physical risks of climate change to their operations, + including the integration of climate scenarios into risk management frameworks and the development of business resilience + strategies. + guidelines: '' +- id: climretr_73_ir_all + text: We search for details on how a company's environmental or sustainability targets are not just internally motivated + but are also aligned with or reference external climate change adaptation goals or targets. This includes looking for + evidence of commitments to internationally recognized frameworks or initiatives, such as the Science Based Targets initiative + (SBTi), which aligns corporate goals with the need to limit global warming to 1.5° C in accordance with the Paris Agreement. + Information indicating participation in renewable energy movements like the RE100 initiative, or adherence to guidelines + and standards set by global organizations, such as the ISO Net Zero Guidelines, is also pertinent. Additionally, we look + for mentions of engaging with suppliers to drive responsible climate choices, aligning carbon footprint reduction efforts + across all scopes of emissions (1, 2, & 3), and setting both near-term and long-term greenhouse gas (GHG) reduction targets. + Evidence of strategies that include investing in nature-based solutions or the management and restoration of natural resources + to combat climate change is also relevant. This comprehensive approach demonstrates a company's broader commitment to + global climate change adaptation and resilience efforts. + guidelines: '' +- id: climretr_74_ir_all + text: We search for details on how a company's environmental or sustainability targets are designed to align with broader + external climate change adaptation and resilience goals, such as those outlined by the Paris Agreement, the Net-Zero Banking + Alliance (NZBA), or the Science Based Targets initiative (SBTi). This includes examining commitments to reduce greenhouse + gas (GHG) emissions, strategies for transitioning to net-zero operations, and efforts to enhance climate change resilience + across their operations and supply chains. Information on specific sector-level targets, especially in carbon-intensive + industries, and the adoption of renewable energy sources to power operations are also pertinent. Additionally, we look + for evidence of the company's engagement with external climate initiatives and frameworks, such as the NZBA or SBTi, and + how these commitments are operationalized within their business strategy. The company's approach to managing physical + and transitional risks associated with climate change, including the financing of projects aligned with a low-carbon future, + is also relevant. + guidelines: '' +- id: climretr_75_ir_all + text: We search for details on the specific measures a company is undertaking or planning to implement in the near future + to minimize waste production. This includes initiatives to adopt circular economy principles, such as enhancing the longevity + of products through design improvements that facilitate reuse, repair, and recycling. Information on efforts to reduce + the use of materials by sourcing them responsibly, increasing the incorporation of recycled content, and eliminating hazardous + substances is pertinent. Additionally, strategies to cut down on operational and technology waste, including the reduction + of single-use plastics and the achievement of waste reduction targets, are of interest. The company's commitment to achieving + certifications that validate their waste reduction efforts, alongside the development of new recycling strategies and + policies aimed at zero waste, are also relevant details. These actions demonstrate the company's short-term commitment + to reducing its environmental impact through waste generation reduction, reflecting its adaptation and resilience strategies + in the face of climate change challenges. + guidelines: '' +- id: climretr_76_ir_all + text: We search for details on how a company plans to collaborate with its downstream partners to address water consumption + and pollution issues. This involves understanding the strategies and initiatives a company has in place to not only reduce + its own water footprint but also to influence and work together with its partners, suppliers, and customers to promote + sustainable water use and pollution control. This includes efforts to replenish water sources, innovate in water-saving + technologies, and engage in policy advocacy for water conservation. Additionally, we look for information on how companies + integrate water sustainability into their investment guidelines, emphasizing the importance of biodiversity, conservation, + and waste management. The goal is to assess the company's commitment to fostering a holistic and collaborative approach + to water sustainability that extends beyond its immediate operations to encompass its entire value chain, thereby enhancing + resilience and adaptation to water-related challenges in the context of climate change. + guidelines: '' +- id: climretr_77_ir_all + text: We search for details on how a company's operations and activities influence the environment, focusing on both direct + and indirect impacts. This includes emissions of greenhouse gases (GHGs) from company operations, energy consumption metrics, + and the effects on land use, such as the extent of land protected versus used for business purposes. It also encompasses + the company's approach to managing its ecological footprint, including strategies for biodiversity and nature conservation. + Additionally, we look at how the company addresses emissions through its value chain, including purchased goods and services, + and efforts to collaborate on or develop low-emission solutions. Information on the company's total GHG emissions, both + direct and indirect, as well as specific figures on electricity consumption, provides insight into its environmental impact. + This analysis helps in understanding the company's commitment to climate change adaptation and resilience, by assessing + its actions to mitigate environmental impacts and enhance sustainability. + guidelines: '' +- id: climretr_78_ir_all + text: We search for details on a company's comprehensive approach towards managing waste, aiming for sustainability and + resilience against climate change. This includes strategies for achieving zero waste through circular economy practices, + such as reducing the use of materials, sourcing responsibly, and enhancing the recyclability and reuse of products and + packaging. Information on specific targets, like diversion rates for operational and construction waste, elimination of + single-use plastics, and the recyclability of products, is crucial. Additionally, the company's efforts in extending the + lifecycle of its products through repair and reuse initiatives, as well as its commitment to reducing waste in its operations, + data centers, and through its product lifecycle, are of interest. The focus is also on how these strategies are integrated + across the company's value chain, from design and material selection to end-of-life management, to mitigate environmental + impacts and promote sustainability across all operations. + guidelines: '' +analysis_guidelines: |- + - Responses must be precise and grounded in report extracts + - Acknowledge uncertainty rather than making assumptions + - Consider potential greenwashing and cheap talk + - Scrutinize quantifiable data vs vague statements + - Maintain a critical analytical tone + - Acknowledge information represents company's perspective + - Evaluate completeness and quality of disclosures + - Use original company wording where possible + - Format responses consistently with bullet points where specified + - Clearly state when no relevant information is found diff --git a/report_analyst/streamlit_app.py b/report_analyst/streamlit_app.py index bf99d569..4c84a5b9 100644 --- a/report_analyst/streamlit_app.py +++ b/report_analyst/streamlit_app.py @@ -1,3 +1,4 @@ +# ruff: noqa: BLE001, E402, RUF001 import asyncio import base64 import html @@ -8,7 +9,7 @@ import time import traceback from pathlib import Path -from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple +from typing import Any, AsyncGenerator, Dict, List, Optional import numpy as np import pandas as pd @@ -30,20 +31,11 @@ # Try to import backend integration features try: - from report_analyst_search_backend.config import ( - BackendConfig, - configure_backend_integration, - display_config_status, - ) - from report_analyst_search_backend.flow_orchestrator import ( - AnalysisResult, - ProcessingResult, - create_flow_orchestrator, - needs_local_analysis, - ) + from report_analyst_search_backend.config import configure_backend_integration + from report_analyst_search_backend.flow_orchestrator import create_flow_orchestrator BACKEND_INTEGRATION_AVAILABLE = True -except ImportError as e: +except ImportError: BACKEND_INTEGRATION_AVAILABLE = False # Configure logging @@ -98,7 +90,11 @@ def is_api_key_missing_message(message: str) -> bool: from report_analyst.core.api_key_manager import APIKeyManager from report_analyst.core.dataframe_manager import ( create_analysis_dataframes, - create_combined_dataframe, +) +from report_analyst.core.llm_models import ( + get_default_llm_model, + get_gemini_models, + get_openai_models, ) from report_analyst.core.prompt_manager import PromptManager from report_analyst.core.question_loader import get_question_loader @@ -110,10 +106,10 @@ def is_api_key_missing_message(message: str) -> bool: # Initialize question loader question_loader = get_question_loader() -# Define model lists based on available API keys -OPENAI_MODELS = ["gpt-4o-mini", "gpt-4o", "gpt-4-turbo", "gpt-3.5-turbo"] - -GEMINI_MODELS = ["gemini-1.5-flash", "gemini-1.5-pro"] +# Model lists — defaults include GPT-5.4 + Gemini 3.x; override via OPENAI_MODELS / GEMINI_MODELS +OPENAI_MODELS = get_openai_models() +GEMINI_MODELS = get_gemini_models() +DEFAULT_LLM_MODEL = get_default_llm_model() def get_available_llm_models() -> List[str]: @@ -217,7 +213,7 @@ def load_question_set(self, question_set: str) -> Dict: } except Exception as e: - logger.error(f"Failed to load questions for {question_set}: {str(e)}") + logger.error(f"Failed to load questions for {question_set}: {e!s}") return {"questions": {}, "name": "", "description": ""} async def analyze_document( @@ -297,13 +293,13 @@ async def analyze_document( yield result except Exception as e: - log_analysis_step(f"Critical error during analysis: {str(e)}", "error") - yield {"error": f"Error analyzing document: {str(e)}"} + log_analysis_step(f"Critical error during analysis: {e!s}", "error") + yield {"error": f"Error analyzing document: {e!s}"} def process_document( self, file_path: str, - selected_questions: List[int] = None, + selected_questions: List[int] | None = None, use_llm_scoring: bool = False, single_call: bool = True, force_recompute: bool = False, @@ -383,7 +379,7 @@ def save_uploaded_file(uploaded_file) -> Optional[str]: else: logger.warning("PostgreSQL file storage not available, falling back to local") except Exception as e: - logger.warning(f"PostgreSQL file storage failed: {str(e)}, falling back to local") + logger.warning(f"PostgreSQL file storage failed: {e!s}, falling back to local") # Fallback to local file storage file_path = Path("temp") / uploaded_file.name @@ -398,8 +394,8 @@ def save_uploaded_file(uploaded_file) -> Optional[str]: st.session_state.file_processed = False return str(file_path) except Exception as e: - logger.error(f"Error saving file: {str(e)}") - st.error(f"Error saving file: {str(e)}") + logger.error(f"Error saving file: {e!s}") + st.error(f"Error saving file: {e!s}") return None @@ -542,7 +538,12 @@ async def analyze_document_and_display( # Update display with cached results logger.info(f"Creating dataframes with cached results for file_key: {file_key}") logger.info( - f"Current session state settings: chunk_size={st.session_state.get('new_chunk_size')}, overlap={st.session_state.get('new_overlap')}, top_k={st.session_state.get('new_top_k')}, llm_model={st.session_state.get('new_llm_model')}, use_llm_scoring={st.session_state.get('new_llm_scoring')}" + f"Current session state settings: " + f"chunk_size={st.session_state.get('new_chunk_size')}, " + f"overlap={st.session_state.get('new_overlap')}, " + f"top_k={st.session_state.get('new_top_k')}, " + f"llm_model={st.session_state.get('new_llm_model')}, " + f"use_llm_scoring={st.session_state.get('new_llm_scoring')}" ) analysis_df, chunks_df = create_analysis_dataframes(st.session_state.results["answers"], file_key) st.session_state.analysis_df = analysis_df @@ -596,7 +597,12 @@ async def analyze_document_and_display( # Update display logger.info(f"Creating dataframes with updated results for file_key: {file_key}") logger.info( - f"Current session state settings: chunk_size={st.session_state.get('new_chunk_size')}, overlap={st.session_state.get('new_overlap')}, top_k={st.session_state.get('new_top_k')}, llm_model={st.session_state.get('new_llm_model')}, use_llm_scoring={st.session_state.get('new_llm_scoring')}" + f"Current session state settings: " + f"chunk_size={st.session_state.get('new_chunk_size')}, " + f"overlap={st.session_state.get('new_overlap')}, " + f"top_k={st.session_state.get('new_top_k')}, " + f"llm_model={st.session_state.get('new_llm_model')}, " + f"use_llm_scoring={st.session_state.get('new_llm_scoring')}" ) analysis_df, chunks_df = create_analysis_dataframes(st.session_state.results["answers"], file_key) @@ -620,9 +626,9 @@ async def analyze_document_and_display( st.session_state.analysis_complete = True except Exception as e: - log_analysis_step(f"Critical error during analysis: {str(e)}", "error") + log_analysis_step(f"Critical error during analysis: {e!s}", "error") log_analysis_step(traceback.format_exc(), "error") - st.error(f"Error during analysis: {str(e)}") + st.error(f"Error during analysis: {e!s}") def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame: @@ -646,8 +652,8 @@ def filter_dataframe(df: pd.DataFrame) -> pd.DataFrame: if is_object_dtype(df[col]): try: df[col] = pd.to_datetime(df[col]) - except Exception: - pass + except (ValueError, TypeError, pd.errors.ParserError): + continue if is_datetime64_any_dtype(df[col]): df[col] = df[col].dt.tz_localize(None) @@ -812,7 +818,7 @@ def get_uploaded_files_history(backend_config=None) -> List[Dict]: return result -def display_analysis_results(analysis_df: pd.DataFrame, chunks_df: pd.DataFrame, file_key: str = None) -> None: +def display_analysis_results(analysis_df: pd.DataFrame, chunks_df: pd.DataFrame, file_key: str | None = None) -> None: """Display analysis results in a consistent format for both individual and consolidated views""" try: if analysis_df.empty: @@ -909,8 +915,8 @@ def display_analysis_results(analysis_df: pd.DataFrame, chunks_df: pd.DataFrame, ) except Exception as e: - logger.error(f"Error displaying analysis results: {str(e)}", exc_info=True) - st.error(f"Error displaying results: {str(e)}") + logger.error(f"Error displaying analysis results: {e!s}", exc_info=True) + st.error(f"Error displaying results: {e!s}") def display_consolidated_results(analyzer, question_set, file_path=None, selected_config=None): @@ -980,7 +986,10 @@ def display_consolidated_results(analyzer, question_set, file_path=None, selecte configs = file_configs[file_path] config_options = [] for config in configs: - label = f"Chunk: {config['chunk_size']}, Overlap: {config['chunk_overlap']}, Top-K: {config['top_k']}, Model: {config['model']}" + label = ( + f"Chunk: {config['chunk_size']}, Overlap: {config['chunk_overlap']}, " + f"Top-K: {config['top_k']}, Model: {config['model']}" + ) config_options.append({"label": label, "config": config}) selected_config = st.selectbox( @@ -1054,7 +1063,8 @@ def display_consolidated_results(analyzer, question_set, file_path=None, selecte # Check if embeddings are available if not analyzer.analyzer.embeddings or analyzer.analyzer.use_backend_llm: st.warning( - "Embeddings not available for similarity search. Using backend mode or embeddings not initialized." + "Embeddings not available for similarity search. " + "Using backend mode or embeddings not initialized." ) query_text = None else: @@ -1076,7 +1086,7 @@ def display_consolidated_results(analyzer, question_set, file_path=None, selecte similarities.append(0.0) # Sort chunks by similarity - chunk_similarity_pairs = list(zip(raw_chunks, similarities)) + chunk_similarity_pairs = list(zip(raw_chunks, similarities, strict=False)) chunk_similarity_pairs.sort(key=lambda x: x[1], reverse=True) # Create rows with similarity scores @@ -1094,9 +1104,9 @@ def display_consolidated_results(analyzer, question_set, file_path=None, selecte st.success(f"✓ Sorted {len(chunks_rows)} chunks by similarity to query") except Exception as e: - st.error(f"Error computing similarity: {str(e)}") + st.error(f"Error computing similarity: {e!s}") logger.error( - f"Error computing similarity: {str(e)}", + f"Error computing similarity: {e!s}", exc_info=True, ) # Fall back to original display @@ -1194,7 +1204,7 @@ def display_consolidated_results(analyzer, question_set, file_path=None, selecte st.warning("No chunks found. Run Step 1 to generate document chunks first.") except Exception as e: - logger.warning(f"Error displaying document chunks with similarity search: {str(e)}") + logger.warning(f"Error displaying document chunks with similarity search: {e!s}") # Continue to show analysis results even if chunk display fails # Get cached results @@ -1241,7 +1251,7 @@ def display_consolidated_results(analyzer, question_set, file_path=None, selecte except Exception as e: logger.error( - f"Error processing result for question {question_id}: {str(e)}", + f"Error processing result for question {question_id}: {e!s}", exc_info=True, ) continue @@ -1260,8 +1270,8 @@ def display_consolidated_results(analyzer, question_set, file_path=None, selecte st.warning("No stored results found for this configuration") except Exception as e: - logger.error(f"Error displaying consolidated results: {str(e)}", exc_info=True) - st.error(f"Error displaying consolidated results: {str(e)}") + logger.error(f"Error displaying consolidated results: {e!s}", exc_info=True) + st.error(f"Error displaying consolidated results: {e!s}") def display_cache_selector(file_path: str): @@ -1294,7 +1304,7 @@ def display_cache_selector(file_path: str): if st.button("Clear Stored Data for File"): try: st.session_state.analyzer.analyzer.cache_manager.clear_cache(file_path) - st.success(f"Stored data cleared for file.") + st.success("Stored data cleared for file.") # Clear results from session state if "results" in st.session_state: del st.session_state.results @@ -1305,17 +1315,17 @@ def display_cache_selector(file_path: str): st.session_state.analysis_complete = False st.rerun() except Exception as e: - st.error(f"Error clearing stored data: {str(e)}") + st.error(f"Error clearing stored data: {e!s}") else: st.info("No stored analyses available for this file") except Exception as e: - st.error(f"Error checking stored data status: {str(e)}") + st.error(f"Error checking stored data status: {e!s}") def get_current_settings(st) -> dict: """Get all current settings from the UI widgets""" # Get first question set as default - default_set = list(question_sets.keys())[0] + default_set = next(iter(question_sets.keys())) available_models = get_available_llm_models() default_model = available_models[0] if available_models else OPENAI_MODELS[0] @@ -1362,7 +1372,7 @@ def update_analyzer_parameters(): st.session_state.new_llm_model = llm_model elif llm_model.startswith("gpt-") and not APIKeyManager.is_configured_key(os.getenv("OPENAI_API_KEY")): logger.error(f"Attempt to use OpenAI model '{llm_model}' without API key") - st.error(f"OPENAI_API_KEY environment variable is not set. OpenAI models will not work correctly.") + st.error("OPENAI_API_KEY environment variable is not set. OpenAI models will not work correctly.") # Update the analyzer with the new parameters try: @@ -1384,7 +1394,7 @@ def update_analyzer_parameters(): logger.info(f"Updated use_llm_scoring to: {st.session_state.use_llm_scoring}") except Exception as e: - st.error(f"Error updating parameters: {str(e)}") + st.error(f"Error updating parameters: {e!s}") async def run_analysis(analyzer, file_path, selected_questions, progress_text): @@ -1402,7 +1412,8 @@ async def run_analysis(analyzer, file_path, selected_questions, progress_text): logger.info(f"[ANALYSIS] Selected questions: {selected_questions}") if "questions" in st.session_state: logger.info( - f"[ANALYSIS] Selected question texts: {[st.session_state.questions[q]['text'] for q in selected_questions if q in st.session_state.questions]}" + "[ANALYSIS] Selected question texts: %s", + [st.session_state.questions[q]["text"] for q in selected_questions if q in st.session_state.questions], ) logger.info( f"[CACHE] Looking up cache for file: {file_path} with config: {config} and questions: {selected_questions}" @@ -1503,8 +1514,8 @@ async def run_analysis(analyzer, file_path, selected_questions, progress_text): progress_text.success("Analysis complete!") except Exception as e: - progress_text.error(f"Error during analysis: {str(e)}") - logger.error(f"Error during analysis: {str(e)}", exc_info=True) + progress_text.error(f"Error during analysis: {e!s}") + logger.error(f"Error during analysis: {e!s}", exc_info=True) def main(): @@ -1521,7 +1532,7 @@ def main(): st.session_state.top_k = 10 # Default number of chunks to retrieve if "llm_model" not in st.session_state: - st.session_state.llm_model = "gpt-4o-mini" # Default model + st.session_state.llm_model = DEFAULT_LLM_MODEL if "question_set" not in st.session_state: st.session_state.question_set = "tcfd" # Default question set @@ -1572,7 +1583,7 @@ def main(): @import url('https://fonts.googleapis.com/css2?family=Afacad:wght@400;500;600;700&display=swap'); @import url('https://fonts.googleapis.com/css2?family=Cousine:wght@400;700&display=swap'); @import url('https://fonts.googleapis.com/icon?family=Material+Icons'); - + /* Material Icons base styles */ .material-icons, i.material-icons { @@ -1592,7 +1603,7 @@ def main(): vertical-align: middle; margin-right: 8px; } - + /* Fix Material Icons rendering issues for Streamlit's stIconMaterial component */ [data-testid="stIconMaterial"] { font-family: 'Material Icons' !important; @@ -1603,7 +1614,7 @@ def main(): text-transform: none !important; letter-spacing: normal !important; } - + /* @font-face fallback for Material Icons */ @font-face { font-family: 'Material Icons'; @@ -1611,7 +1622,7 @@ def main(): font-weight: 400; src: url(https://fonts.gstatic.com/s/materialicons/v142/flUhRq6tzZclQEJ-Vdg-IuiaDsNc.woff2) format('woff2'); } - + /* Add Material Icon to stAlert elements - only ONE icon per alert */ /* Add icon only to the markdown container, NOT to paragraphs to avoid duplicates */ [data-testid="stAlert"] [data-testid="stMarkdownContainer"]::before { @@ -1622,13 +1633,13 @@ def main(): margin-right: 8px; display: inline-block; } - + /* Remove icons from paragraphs inside stAlert to prevent double icons */ [data-testid="stAlert"] p::before { content: none !important; display: none !important; } - + /* Add icons to custom notifications */ [data-testid="stNotification"] [data-testid="stMarkdownContainer"]::before { content: 'info'; @@ -1638,7 +1649,7 @@ def main(): margin-right: 8px; display: inline-block; } - + /* Remove icons from paragraphs in custom notifications too */ [data-testid="stNotification"] p::before { content: none !important; @@ -1675,7 +1686,7 @@ def main(): .api-key-missing-alert-message { display: block; } - + /* Settings expander icon in sidebar */ [data-testid="stSidebar"] [data-testid="stExpander"] summary::before { content: 'settings'; @@ -1685,48 +1696,48 @@ def main(): margin-right: 8px; display: inline-block; } - + /* Active navigation item - light purple background with dark purple text */ [data-testid="stSidebar"] .nav-link-selected { background-color: rgba(67, 19, 200, 0.15) !important; color: #4313C8 !important; font-weight: 700 !important; } - + /* Active navigation item icon - dark purple */ [data-testid="stSidebar"] .nav-link-selected i { color: #4313C8 !important; } - + /* Inactive navigation items - gray text and icons */ [data-testid="stSidebar"] .nav-link:not(.nav-link-selected) { color: #7872A7 !important; } - + [data-testid="stSidebar"] .nav-link:not(.nav-link-selected) i { color: #7872A7 !important; } - + /* Designer Colors - Exact specifications from Daniela */ - + /* ========== LIGHT MODE ========== */ - + /* Main app background - #F5F7FF */ .stApp { background-color: #F5F7FF !important; font-family: 'Afacad', sans-serif !important; } - + /* Primary font - Afacad for titles and body text */ body, .main, p, span, div, label { font-family: 'Afacad', sans-serif !important; } - + /* Titles use Afacad */ h1, h2, h3, h4, h5, h6 { font-family: 'Afacad', sans-serif !important; } - + /* Secondary font - Cousine for UI elements */ button, .stButton > button, input, textarea, select, @@ -1739,12 +1750,12 @@ def main(): code, pre { font-family: 'Cousine', monospace !important; } - + /* Main container - #FFFFFF */ .main .block-container { background-color: #FFFFFF !important; } - + /* Secondary containers - C0C4FA 10% opacity */ [data-testid="stExpander"], .stAlert, @@ -1752,70 +1763,71 @@ def main(): .stInfo { background-color: rgba(192, 196, 250, 0.1) !important; } - + /* Fix text layout - prevent vertical stacking */ .stInfo { word-break: normal !important; white-space: normal !important; } - + .stInfo p, .stInfo span { writing-mode: horizontal-tb !important; text-orientation: mixed !important; } - + /* Ensure columns don't cause vertical text */ [data-testid="column"] { min-width: 0 !important; } - + [data-testid="column"] * { word-break: normal !important; white-space: normal !important; } - + /* Titles - #4313C8 */ h1, h2, [data-testid="stMarkdownContainer"] h1, [data-testid="stMarkdownContainer"] h2 { color: #4313C8 !important; } - + /* Subtitles - #979DF6 */ h3, h4, [data-testid="stMarkdownContainer"] h3, [data-testid="stMarkdownContainer"] h4 { color: #979DF6 !important; } - + /* Body text - #170843 */ p, span, label { color: #170843 !important; } - + /* Don't force color on all divs - let them inherit to prevent layout issues */ div:not([data-testid="stSidebar"] div):not(.stCheckbox):not([data-testid="stMarkdownContainer"]) { color: #170843 !important; } - + /* Caption text - #718096 */ .stCaption, small, [data-testid="stCaptionContainer"] { color: #718096 !important; } - + /* Sidebar - white background */ [data-testid="stSidebar"] { background-color: #FFFFFF !important; } - + /* Sidebar text - #7872A7 (exclude option-menu navigation) */ - [data-testid="stSidebar"] *:not([data-testid="stSidebarNav"] [aria-current="page"] *):not(.nav-link):not(.nav-link-selected):not(.nav-link *):not([class*="nav-link"]) { + [data-testid="stSidebar"] *:not([data-testid="stSidebarNav"] [aria-current="page"] *) +:not(.nav-link):not(.nav-link-selected):not(.nav-link *):not([class*="nav-link"]) { color: #7872A7 !important; } - + /* Ensure option-menu navigation styles are not overridden */ [data-testid="stSidebar"] .nav-link, [data-testid="stSidebar"] .nav-link-selected { color: inherit !important; } - + /* File Display Panel - Unique class for green panel styling */ /* The key="file-display-panel" creates the class st-key-file-display-panel */ /* Target the container element which has the st-key- class */ @@ -1826,7 +1838,7 @@ def main(): padding: 1rem 1.5rem !important; margin: 1rem 0 1.5rem 0 !important; } - + /* Target the horizontal block inside the container (for columns) */ .st-key-file-display-panel [data-testid="stHorizontalBlock"] { background-color: transparent !important; @@ -1836,12 +1848,12 @@ def main(): .st-key-file-display-panel [data-testid="column"] { background-color: transparent !important; } - + /* Keep upload date gray */ .st-key-file-display-panel .pdf-upload-date { color: #718096 !important; } - + .pdf-icon-box { background-color: #C8E6C9; border-radius: 12px; @@ -1852,53 +1864,53 @@ def main(): justify-content: center; flex-shrink: 0; } - + .pdf-icon-box .material-icons, .pdf-icon-box i.material-icons { font-size: 28px !important; color: #2E7D32 !important; display: inline-block !important; } - + .pdf-info-section { flex-grow: 1; } - + .pdf-upload-date { font-size: 13px; color: #718096; display: block; margin-top: 4px; } - + /* Style ONLY the PDF selectbox - target it specifically within the file display panel */ /* Make the selectbox container bigger */ .st-key-file-display-panel [data-baseweb="select"] { min-width: 300px !important; max-width: 500px !important; } - + /* Target the main selectbox wrapper */ .st-key-file-display-panel [data-baseweb="select"] > div { background-color: transparent !important; border: none !important; box-shadow: none !important; } - + /* Target the div with value attribute (the displayed text) - make it bigger, bolder, and green */ .st-key-file-display-panel [data-baseweb="select"] div[value] { font-size: 22px !important; font-weight: 800 !important; color: #1B9E6B !important; } - + /* Also target by the specific class pattern for the value div */ .st-key-file-display-panel [data-baseweb="select"] [class*="st-dn"] { font-size: 22px !important; font-weight: 800 !important; color: #1B9E6B !important; } - + /* Target nested divs that contain the text */ .st-key-file-display-panel [data-baseweb="select"] > div > div > div[value], .st-key-file-display-panel [data-baseweb="select"] > div > div > div[class*="st-dn"] { @@ -1906,21 +1918,21 @@ def main(): font-weight: 800 !important; color: #1B9E6B !important; } - + /* Make the dropdown arrow bigger, bold, and green */ .st-key-file-display-panel [data-baseweb="select"] svg { color: #1B9E6B !important; width: 28px !important; height: 28px !important; } - + .st-key-file-display-panel [data-baseweb="select"] svg path, .st-key-file-display-panel [data-baseweb="select"] svg polygon { stroke-width: 4 !important; stroke: #1B9E6B !important; fill: #1B9E6B !important; } - + /* Question Set Display Panel - same styling as file display panel */ [data-testid="stVerticalBlock"].st-key-question-set-display-panel, .st-key-question-set-display-panel[data-testid="stVerticalBlock"] { @@ -1929,26 +1941,26 @@ def main(): padding: 1rem 1.5rem !important; margin: 1rem 0 0.5rem 0 !important; } - + .st-key-question-set-display-panel [data-testid="stHorizontalBlock"] { background-color: transparent !important; } - + .st-key-question-set-display-panel [data-testid="column"] { background-color: transparent !important; } - + .st-key-question-set-display-panel [data-baseweb="select"] { min-width: 300px !important; max-width: 500px !important; } - + .st-key-question-set-display-panel [data-baseweb="select"] > div { background-color: transparent !important; border: none !important; box-shadow: none !important; } - + .st-key-question-set-display-panel [data-baseweb="select"] div[value], .st-key-question-set-display-panel [data-baseweb="select"] [class*="st-dn"], .st-key-question-set-display-panel [data-baseweb="select"] > div > div > div[value], @@ -1957,20 +1969,20 @@ def main(): font-weight: 800 !important; color: #1B9E6B !important; } - + .st-key-question-set-display-panel [data-baseweb="select"] svg { color: #1B9E6B !important; width: 28px !important; height: 28px !important; } - + .st-key-question-set-display-panel [data-baseweb="select"] svg path, .st-key-question-set-display-panel [data-baseweb="select"] svg polygon { stroke-width: 4 !important; stroke: #1B9E6B !important; fill: #1B9E6B !important; } - + /* Styled Selectboxes - White background, thin border */ /* Only target selectboxes that are NOT in the PDF container */ /* Target selectboxes inside expanders or other sections, but NOT in PDF container */ @@ -1980,12 +1992,12 @@ def main(): border: 1px solid #E2E8F0 !important; border-radius: 4px !important; } - + /* Ensure selectboxes in expanders have normal text color (NOT green) */ [data-testid="stExpander"] [data-baseweb="select"] > div > div > div { color: #170843 !important; } - + /* Ensure selectbox arrows in expanders are NOT green */ [data-testid="stExpander"] [data-baseweb="select"] svg path, [data-testid="stExpander"] [data-baseweb="select"] svg polygon { @@ -1993,12 +2005,12 @@ def main(): fill: #170843 !important; stroke-width: 1 !important; } - + [data-testid="stExpander"] [data-baseweb="select"]:hover > div, [data-testid="stExpander"] [data-baseweb="select"]:hover > div > div { border-color: #4313C8 !important; } - + /* Styled Questions Table */ .questions-table-container { background-color: #FFFFFF; @@ -2007,7 +2019,7 @@ def main(): padding: 1rem; margin: 1rem 0; } - + /* Sidebar accent (active item) - #4313C8 with white text */ [data-testid="stSidebarNav"] li[aria-selected="true"], [data-testid="stSidebarNav"] a[aria-selected="true"], @@ -2016,21 +2028,21 @@ def main(): background-color: #4313C8 !important; border-radius: 4px !important; } - + /* Active sidebar item text and icons - white */ [data-testid="stSidebarNav"] li[aria-current="page"] *, [data-testid="stSidebarNav"] a[aria-current="page"] * { color: #ffffff !important; fill: #ffffff !important; } - + /* Sidebar navigation radio buttons - styled like screen design */ [data-testid="stSidebar"] [data-baseweb="radio"] { display: flex !important; flex-direction: column !important; gap: 4px !important; } - + /* Hide radio button input circles completely */ [data-testid="stSidebar"] [data-baseweb="radio"] input[type="radio"] { display: none !important; @@ -2042,14 +2054,14 @@ def main(): margin: 0 !important; padding: 0 !important; } - + /* Hide the radio button circle indicators */ [data-testid="stSidebar"] [data-baseweb="radio"] > div > div:first-child, [data-testid="stSidebar"] [data-baseweb="radio"] label::before, [data-testid="stSidebar"] [data-baseweb="radio"] label > div:first-child:not(span) { display: none !important; } - + [data-testid="stSidebar"] [data-baseweb="radio"] > label { padding: 10px 15px !important; border-radius: 6px !important; @@ -2062,22 +2074,22 @@ def main(): align-items: center !important; gap: 8px !important; } - + [data-testid="stSidebar"] [data-baseweb="radio"] > label:hover { background-color: rgba(67, 19, 200, 0.1) !important; } - + /* Inactive sidebar items - purple text */ [data-testid="stSidebar"] [data-baseweb="radio"] label { color: #4313C8 !important; } - + [data-testid="stSidebar"] [data-baseweb="radio"] label span { color: #4313C8 !important; font-family: 'Cousine', monospace !important; font-weight: 400 !important; } - + /* Active/selected sidebar item - purple background with white text */ /* Streamlit uses a div wrapper with data-checked attribute */ [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] > label, @@ -2090,13 +2102,13 @@ def main(): border-radius: 6px !important; font-weight: 700 !important; } - + /* Also target the parent div when checked */ [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] { background-color: #4313C8 !important; border-radius: 6px !important; } - + /* Active sidebar item text - white and bold */ [data-testid="stSidebar"] [data-baseweb="radio"] input[type="radio"]:checked ~ label span, [data-testid="stSidebar"] [data-baseweb="radio"] input[type="radio"]:checked + label span, @@ -2108,7 +2120,7 @@ def main(): color: #ffffff !important; font-weight: 700 !important; } - + /* Active sidebar item - also target the parent container */ [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] > label, [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] label { @@ -2116,13 +2128,13 @@ def main(): color: #ffffff !important; font-weight: 700 !important; } - + [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] > label span, [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] label span { color: #ffffff !important; font-weight: 700 !important; } - + /* Sidebar Material Icons - match text color */ [data-testid="stSidebar"] [data-baseweb="radio"] label .nav-material-icon, [data-testid="stSidebar"] [data-baseweb="radio"] label .material-icons { @@ -2131,7 +2143,7 @@ def main(): margin-right: 8px !important; vertical-align: middle !important; } - + /* Active sidebar item Material Icons - white */ [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] > label .nav-material-icon, [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] label .nav-material-icon, @@ -2140,7 +2152,7 @@ def main(): [data-testid="stSidebar"] [data-baseweb="radio"] input[type="radio"]:checked + label .nav-material-icon { color: #ffffff !important; } - + /* Keep tooltip icons (help icons) visible and styled */ [data-testid="stSidebar"] [data-baseweb="radio"] label [data-testid="stTooltipIcon"] svg, [data-testid="stSidebar"] [data-baseweb="radio"] label [data-testid="stTooltipHoverTarget"] svg { @@ -2148,27 +2160,29 @@ def main(): color: #4313C8 !important; stroke: #4313C8 !important; } - - [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] > label [data-testid="stTooltipIcon"] svg, - [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] label [data-testid="stTooltipIcon"] svg { + + [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] > label + [data-testid="stTooltipIcon"] svg, + [data-testid="stSidebar"] [data-baseweb="radio"] > div[data-checked="true"] label + [data-testid="stTooltipIcon"] svg { color: #ffffff !important; stroke: #ffffff !important; } - + /* Green accent - #2E9D6F */ .stSuccess { background-color: rgba(46, 157, 111, 0.3) !important; border-color: #2E9D6F !important; color: #2E9D6F !important; } - + /* Green cards - 30% and 10% opacity */ [data-testid="stNotification"][data-status="success"] { background-color: rgba(46, 157, 111, 0.1) !important; } - + /* ========== UNIFIED BUTTON STYLES ========== */ - + /* Help icon button - styled as icon only, no background */ div[data-testid="stButton"] button:has-text("ℹ️") { background-color: transparent !important; @@ -2183,14 +2197,14 @@ def main(): box-shadow: none !important; line-height: 1 !important; } - + div[data-testid="stButton"] button:has-text("ℹ️"):hover { background-color: transparent !important; color: #4313C8 !important; opacity: 0.7 !important; transform: none !important; } - + /* Select All button - small light purple button like processing steps */ /* Target primary buttons that appear after "Select Questions" heading */ h3:has-text("Select Questions") + div[data-testid="stButton"] button[kind="primary"], @@ -2208,14 +2222,14 @@ def main(): width: auto !important; max-width: fit-content !important; } - + /* Hover state for Select All button */ h3:has-text("Select Questions") + div[data-testid="stButton"] button[kind="primary"]:hover, div[data-testid="stButton"] button[kind="primary"]:hover { background-color: rgba(192, 196, 250, 0.2) !important; border-color: #4313C8 !important; } - + /* Override for larger primary buttons (like Analyze Selected Questions, Reanalyze) */ /* These buttons have more text, so we can target them by their longer text content */ div[data-testid="stButton"]:has(button:contains("Analyze")) button, @@ -2227,7 +2241,7 @@ def main(): width: 100% !important; max-width: 100% !important; } - + /* Default: All buttons in main content - white background with purple text (like Browse File) */ /* Style all buttons first, then override for sidebar and special buttons */ .stButton > button, @@ -2249,7 +2263,7 @@ def main(): width: auto !important; min-width: auto !important; } - + .stButton > button:hover, .stDownloadButton > button:hover, [data-testid="stDownloadButton"] button:hover, @@ -2260,7 +2274,7 @@ def main(): border: 1px solid #4313C8 !important; box-shadow: 0 2px 6px rgba(67, 19, 200, 0.25), 0 1px 3px rgba(67, 19, 200, 0.15) !important; } - + .stButton > button:active, .stButton > button:focus, .stDownloadButton > button:active, @@ -2277,7 +2291,7 @@ def main(): outline: none !important; box-shadow: 0 1px 3px rgba(67, 19, 200, 0.3) !important; } - + /* Sidebar buttons - override with purple background (higher specificity) */ [data-testid="stSidebar"] .stButton > button { background-color: #4313C8 !important; @@ -2290,14 +2304,14 @@ def main(): font-weight: 400 !important; padding: 0.5rem 1rem !important; } - + [data-testid="stSidebar"] .stButton > button:hover { background-color: #979DF6 !important; color: #ffffff !important; border: 2px solid #979DF6 !important; box-shadow: none !important; } - + [data-testid="stSidebar"] .stButton > button:active, [data-testid="stSidebar"] .stButton > button:focus { background-color: #4313C8 !important; @@ -2306,17 +2320,18 @@ def main(): outline: none !important; box-shadow: none !important; } - + /* Remove all orange/red borders and states from buttons */ /* Note: Main button styles are defined above, this just ensures border color */ /* Exclude sidebar and file uploader buttons, but include download buttons */ button:not([data-testid="stSidebar"] button):not([data-testid*="FileUploader"] button), .stButton > button:not([data-testid="stSidebar"] .stButton > button):not([data-testid*="FileUploader"] button), - [data-baseweb="button"]:not([data-testid="stSidebar"] [data-baseweb="button"]):not([data-testid*="FileUploader"] [data-baseweb="button"]) { + [data-baseweb="button"]:not([data-testid="stSidebar"] [data-baseweb="button"]) + :not([data-testid*="FileUploader"] [data-baseweb="button"]) { border-color: #4313C8 !important; outline: none !important; } - + /* File uploader buttons - purple with white text (keep special styling) */ [data-testid="stFileUploader"] button, [data-testid="stFileUploader"] [data-baseweb="button"], @@ -2330,7 +2345,7 @@ def main(): font-weight: 400 !important; padding: 0.5rem 1rem !important; } - + [data-testid="stFileUploader"] button:hover, [data-testid="stFileUploader"] [data-baseweb="button"]:hover, .stFileUploader button:hover { @@ -2338,7 +2353,7 @@ def main(): color: #ffffff !important; border: 2px solid #979DF6 !important; } - + [data-testid="stFileUploader"] button:active, [data-testid="stFileUploader"] button:focus, [data-testid="stFileUploader"] [data-baseweb="button"]:active, @@ -2349,10 +2364,10 @@ def main(): border: 2px solid #4313C8 !important; outline: none !important; } - + /* Download buttons - use main content button style (white background, purple text) */ /* They inherit from .stButton > button above, no special override needed */ - + /* Secondary buttons - transparent with purple border */ button[data-baseweb="button"][kind="secondary"], [data-baseweb="button"][kind="secondary"], @@ -2363,14 +2378,14 @@ def main(): border-radius: 6px !important; font-family: 'Cousine', monospace !important; } - + button[data-baseweb="button"][kind="secondary"]:hover, [data-baseweb="button"][kind="secondary"]:hover, button.kind-secondary:hover { background-color: #4313C8 !important; color: #ffffff !important; } - + /* Checkboxes - purple accent, remove ALL orange, make checkmark visible */ .stCheckbox > label > span[data-baseweb="checkbox"], span[data-baseweb="checkbox"], @@ -2382,7 +2397,7 @@ def main(): width: 18px !important; height: 18px !important; } - + .stCheckbox > label > span[data-baseweb="checkbox"][aria-checked="true"], span[data-baseweb="checkbox"][aria-checked="true"], [data-baseweb="checkbox"][aria-checked="true"], @@ -2390,7 +2405,7 @@ def main(): background-color: #4313C8 !important; border-color: #4313C8 !important; } - + /* Make checkmark visible - white checkmark on purple background */ .stCheckbox > label > span[data-baseweb="checkbox"][aria-checked="true"] svg, span[data-baseweb="checkbox"][aria-checked="true"] svg, @@ -2402,7 +2417,7 @@ def main(): visibility: visible !important; opacity: 1 !important; } - + /* Alternative checkmark using CSS if SVG doesn't work */ .stCheckbox > label > span[data-baseweb="checkbox"][aria-checked="true"]::after, span[data-baseweb="checkbox"][aria-checked="true"]::after { @@ -2417,7 +2432,7 @@ def main(): transform: translate(-50%, -50%) !important; line-height: 1 !important; } - + /* Make checkmark visible in Streamlit's internal checkboxes */ span.st-bi[aria-checked="true"] svg, span[class*="st-bi"][aria-checked="true"] svg { @@ -2427,19 +2442,19 @@ def main(): visibility: visible !important; opacity: 1 !important; } - + /* Question checkboxes - make them visible like in screen design */ .stCheckbox { margin-bottom: 12px !important; width: 100% !important; max-width: 100% !important; } - + .stCheckbox > div { width: 100% !important; max-width: 100% !important; } - + .stCheckbox label { display: flex !important; flex-direction: row !important; @@ -2450,7 +2465,7 @@ def main(): font-family: 'Cousine', monospace !important; box-sizing: border-box !important; } - + .stCheckbox label > span[data-baseweb="checkbox"] { min-width: 18px !important; width: 18px !important; @@ -2463,7 +2478,7 @@ def main(): opacity: 1 !important; margin-top: 2px !important; } - + /* Fix markdown container - ensure horizontal text and proper responsive layout */ .stCheckbox label [data-testid="stMarkdownContainer"] { writing-mode: horizontal-tb !important; @@ -2478,7 +2493,7 @@ def main(): overflow: visible !important; box-sizing: border-box !important; } - + .stCheckbox label [data-testid="stMarkdownContainer"] p { writing-mode: horizontal-tb !important; text-orientation: mixed !important; @@ -2498,7 +2513,7 @@ def main(): word-wrap: break-word !important; box-sizing: border-box !important; } - + /* Remove any background or border from checkbox label elements */ .stCheckbox label, .stCheckbox label *, @@ -2512,7 +2527,7 @@ def main(): outline: none !important; box-shadow: none !important; } - + /* Remove borders from markdown container specifically */ .stCheckbox label [data-testid="stMarkdownContainer"], .stCheckbox label [data-testid="stMarkdownContainer"] *, @@ -2523,7 +2538,7 @@ def main(): box-shadow: none !important; background-color: transparent !important; } - + /* Prevent text fragmentation in checkbox labels */ .stCheckbox label [data-testid="stMarkdownContainer"] * { word-break: normal !important; @@ -2532,7 +2547,7 @@ def main(): background-color: transparent !important; border: none !important; } - + /* Ensure checkbox container doesn't break text */ .stCheckbox > div, .stCheckbox > div > div { @@ -2541,17 +2556,17 @@ def main(): background-color: transparent !important; border: none !important; } - + /* Remove borders from all checkbox-related elements */ .stCheckbox * { border: none !important; } - + /* But keep the checkbox itself visible */ .stCheckbox label > span[data-baseweb="checkbox"] { border: 2px solid #4313C8 !important; } - + /* Ensure all checkboxes are visible */ input[type="checkbox"] { width: 18px !important; @@ -2560,7 +2575,7 @@ def main(): opacity: 1 !important; display: block !important; } - + /* Remove orange from Streamlit's internal checkbox elements */ span.st-bi, span[class*="st-bi"], @@ -2578,13 +2593,13 @@ def main(): background-color: transparent !important; border: 2px solid #4313C8 !important; } - + span.st-bi[aria-checked="true"], span[class*="st-bi"][aria-checked="true"] { background-color: #4313C8 !important; border-color: #4313C8 !important; } - + /* Make checkmark visible in Streamlit's internal checkboxes */ span.st-bi[aria-checked="true"]::after, span[class*="st-bi"][aria-checked="true"]::after { @@ -2594,12 +2609,12 @@ def main(): font-weight: bold !important; display: block !important; } - + /* Force remove #FF4B4B (Streamlit's default orange) from ALL elements */ * { --primary-color: #4313C8 !important; } - + /* Remove orange from ALL elements with #FF4B4B */ div[style*="#FF4B4B"], span[style*="#FF4B4B"], @@ -2613,7 +2628,7 @@ def main(): border-color: #4313C8 !important; color: #4313C8 !important; } - + /* Remove orange from Streamlit's internal div classes */ div[class*="st-cu"], div[class*="st-cl"], @@ -2629,51 +2644,51 @@ def main(): background-color: transparent !important; border-color: transparent !important; } - + /* Specifically hide the orange line element */ div.st-cu.st-cl.st-f6.st-f7.st-f8.st-f9.st-fo.st-fp.st-b0.st-fq.st-fr { display: none !important; background-color: transparent !important; border-color: transparent !important; } - + /* Radio buttons - purple accent */ .stRadio > label > div[data-baseweb="radio"] > div { background-color: transparent !important; border-color: #4313C8 !important; } - + .stRadio > label > div[data-baseweb="radio"][aria-checked="true"] > div:first-child { background-color: #4313C8 !important; } - + /* Number input buttons */ .stNumberInput button { color: #4313C8 !important; background-color: transparent !important; } - + .stNumberInput button:hover { background-color: rgba(67, 19, 200, 0.1) !important; } - + /* Tabs - remove orange/red underline completely */ .stTabs [data-baseweb="tab"] { color: #170843 !important; } - + .stTabs [aria-selected="true"], .stTabs [aria-selected="true"] [data-baseweb="tab"] { color: #4313C8 !important; border-bottom-color: #4313C8 !important; } - + /* Remove all orange/red Streamlit defaults from tabs */ [data-baseweb="tab"][aria-selected="true"], [data-baseweb="tab-list"] [aria-selected="true"] { border-bottom: 2px solid #4313C8 !important; } - + /* Remove orange from tab indicators and underlines */ .stTabs [aria-selected="true"]::after, .stTabs [aria-selected="true"]::before, @@ -2682,57 +2697,57 @@ def main(): background-color: #4313C8 !important; border-color: #4313C8 !important; } - + /* Target Streamlit's internal tab styling */ div[class*="stTabs"] [aria-selected="true"], div[class*="stTabs"] [aria-selected="true"] > div { border-bottom-color: #4313C8 !important; } - + /* Remove any orange borders/lines from tabs */ .stTabs * { border-color: transparent !important; } - + .stTabs [aria-selected="true"] * { border-bottom-color: #4313C8 !important; } - + /* Progress bars */ .stProgress > div > div > div { background-color: #4313C8 !important; } - + /* Sliders */ [data-baseweb="slider"] [data-baseweb="slider-track"] { background-color: #4313C8 !important; } - + [data-baseweb="slider"] [data-baseweb="slider-handle"] { background-color: #4313C8 !important; border-color: #4313C8 !important; } - + /* File uploader */ [data-testid="stFileUploader"] button { background-color: #4313C8 !important; color: #ffffff !important; } - + /* Remove any orange from links */ a:link, a:visited { color: #4313C8 !important; } - + a:hover { color: #979DF6 !important; } - + /* Expander icons */ .streamlit-expanderHeader { color: #4313C8 !important; } - + /* Header - remove orange/red bar at top */ [data-testid="stHeader"], [data-testid="stHeader"] > div, @@ -2740,14 +2755,14 @@ def main(): background-color: transparent !important; border-bottom: none !important; } - + /* Remove orange from progress bars */ [data-baseweb="progressbar"], [data-baseweb="progressbar"] > div, [data-baseweb="progressbar"] > div > div { background-color: #4313C8 !important; } - + /* Remove orange from any remaining Streamlit elements */ [style*="rgb(255, 75, 75)"], [style*="rgb(255, 107, 107)"], @@ -2761,7 +2776,7 @@ def main(): background-color: #4313C8 !important; border-color: #4313C8 !important; } - + /* Force remove orange backgrounds */ div[style*="background"][style*="255, 75"], div[style*="background"][style*="255, 107"], @@ -2769,12 +2784,12 @@ def main(): div[style*="background"][style*="#ff6b"] { background-color: transparent !important; } - + /* Links */ a { color: #4313C8 !important; } - + /* Footer styling - in sidebar at bottom */ [data-testid="stSidebar"] .footer { text-align: center; @@ -2783,11 +2798,11 @@ def main(): border-top: 1px solid rgba(67, 19, 200, 0.1); margin-top: 20px; } - + [data-testid="stSidebar"] .footer a { color: #4313C8 !important; } - + [data-testid="stSidebar"] .footer img { height: 25px; max-width: 100%; @@ -2796,25 +2811,25 @@ def main(): margin-right: 8px; object-fit: contain; } - + [data-testid="stSidebar"] .footer { overflow: visible; word-wrap: break-word; } - + [data-testid="stSidebar"] .footer p { margin: 4px 0; color: #7872A7; font-size: 11px; } - + """ st.markdown(custom_css, unsafe_allow_html=True) except Exception as e: # Fallback if theme detection fails - logger.warning(f"Could not apply custom theme: {str(e)}") + logger.warning(f"Could not apply custom theme: {e!s}") # Initialize analyzer with default question set try: @@ -2824,7 +2839,7 @@ def main(): analyzer = st.session_state.analyzer # Use the stored analyzer except Exception as e: - st.error(f"Error initializing analyzer: {str(e)}") + st.error(f"Error initializing analyzer: {e!s}") st.exception(e) return @@ -2837,15 +2852,15 @@ def main(): st.sidebar.markdown( f"""
- Open Sustainability Analyst
""", unsafe_allow_html=True, ) except Exception as e: - logger.warning(f"Could not load sidebar logo: {str(e)}") + logger.warning(f"Could not load sidebar logo: {e!s}") # Create sidebar navigation using streamlit-option-menu st.sidebar.markdown("---") @@ -2861,9 +2876,10 @@ def main(): "Upload Report", "Report Analyst", "All Results", + "Benchmarking", "Settings", ], - icons=["house", "file-text", "bar-chart", "gear"], + icons=["house", "file-text", "bar-chart", "target", "gear"], menu_icon=None, default_index=0, orientation="vertical", @@ -2893,7 +2909,7 @@ def main(): ) except ImportError: # Fallback to regular radio if package not installed - nav_options = ["Upload Report", "Report Analyst", "All Results", "Settings"] + nav_options = ["Upload Report", "Report Analyst", "All Results", "Benchmarking", "Settings"] nav_page = st.sidebar.radio("", nav_options, key="nav_page", label_visibility="collapsed") # Show page-specific content based on navigation @@ -2963,7 +2979,7 @@ def main(): if not has_env_openai or override_openai: # Track previous values to detect changes - prev_openai_key = st.session_state.get("prev_openai_key", current_openai_key) + st.session_state.get("prev_openai_key", current_openai_key) # OpenAI API Key input openai_key_input = st.text_input( @@ -3016,7 +3032,7 @@ def main(): if not has_env_google or override_google: # Track previous values to detect changes - prev_google_key = st.session_state.get("prev_google_key", current_google_key) + st.session_state.get("prev_google_key", current_google_key) # Google/Gemini API Key input google_key_input = st.text_input( @@ -3024,7 +3040,10 @@ def main(): value="", # Never show the actual key in the input type="password", key="google_api_key_input", - help="Enter your Google API key to use Gemini models. Leave empty to use existing key from environment.", + help=( + "Enter your Google API key to use Gemini models. " + "Leave empty to use existing key from environment." + ), placeholder=("Enter your Google API key" if not current_google_key else "Enter new key to update"), ) @@ -3075,7 +3094,8 @@ def main(): database_url = f"sqlite:///{db_path}" database_type = "SQLite" st.info( - f"**Type:** {database_type}\n\n**Path:** `{db_path}`\n\n*Configure via `STORAGE_PATH` environment variable*" + f"**Type:** {database_type}\n\n**Path:** `{db_path}`\n\n" + f"*Configure via `STORAGE_PATH` environment variable*" ) else: # Parse PostgreSQL URL to show connection details (masked) @@ -3119,19 +3139,25 @@ def main(): db = "?" st.info( - f"**Type:** {database_type}\n\n**Host:** `{host}`\n**Port:** `{port}`\n**Database:** `{db}`\n**User:** `{user}`\n\n*Configure via `DATABASE_URL` environment variable*" + f"**Type:** {database_type}\n\n" + f"**Host:** `{host}`\n**Port:** `{port}`\n" + f"**Database:** `{db}`\n**User:** `{user}`\n\n" + f"*Configure via `DATABASE_URL` environment variable*" ) else: st.info( - f"**Type:** {database_type}\n\n**Connection:** `{masked_url}`\n\n*Configure via `DATABASE_URL` environment variable*" + f"**Type:** {database_type}\n\n**Connection:** `{masked_url}`\n\n" + f"*Configure via `DATABASE_URL` environment variable*" ) else: st.info( - f"**Type:** {database_type}\n\n**Connection:** `{masked_url}`\n\n*Configure via `DATABASE_URL` environment variable*" + f"**Type:** {database_type}\n\n**Connection:** `{masked_url}`\n\n" + f"*Configure via `DATABASE_URL` environment variable*" ) except Exception: st.info( - f"**Type:** {database_type}\n\n**Connection:** `{masked_url}`\n\n*Configure via `DATABASE_URL` environment variable*" + f"**Type:** {database_type}\n\n**Connection:** `{masked_url}`\n\n" + f"*Configure via `DATABASE_URL` environment variable*" ) # Store in session state for use by DocumentAnalyzer @@ -3221,7 +3247,10 @@ def main(): "Store files in PostgreSQL", value=st.session_state.get("postgres_file_storage_enabled", False), key="use_postgres_file_storage", - help="Store uploaded files in PostgreSQL database (useful for Heroku deployments). Files are stored as BYTEA/BLOB. This is an enterprise feature.", + help=( + "Store uploaded files in PostgreSQL database (useful for Heroku deployments). " + "Files are stored as BYTEA/BLOB. This is an enterprise feature." + ), ) # Store in a separate key that persists across page navigation st.session_state.postgres_file_storage_enabled = use_postgres_storage @@ -3420,7 +3449,9 @@ def main(): Processing Steps help_outline
- You can choose if you want to first only cut the report in pieces (Chunking), make it searchable (Embedding), map text to questions (Question Mapping), or answer the questions (Question Answering). Note: Answering questions incurs LLM API costs. + You can choose if you want to first only cut the report in pieces (Chunking), + make it searchable (Embedding), map text to questions (Question Mapping), or answer + the questions (Question Answering). Note: Answering questions incurs LLM API costs.
""", @@ -3456,16 +3487,16 @@ def main(): try: step_status = analyzer.analyzer.check_step_completion(selected_uri) except Exception as e: - logger.warning(f"Error checking step completion: {str(e)}") + logger.warning(f"Error checking step completion: {e!s}") else: file_path_for_status = Path(selected_file_obj["path"]) if file_path_for_status.exists(): try: step_status = analyzer.analyzer.check_step_completion(str(file_path_for_status)) except Exception as e: - logger.warning(f"Error checking step completion: {str(e)}") + logger.warning(f"Error checking step completion: {e!s}") except Exception as e: - logger.warning(f"Error getting step status: {str(e)}") + logger.warning(f"Error getting step status: {e!s}") # Define processing steps with shorter labels step_options = ["Chunk", "Embed", "Map", "Answer"] @@ -3494,7 +3525,7 @@ def main(): font-size: 14px !important; font-family: 'Afacad', sans-serif !important; } - + /* Make all slider tick labels visible */ [data-baseweb="slider"] [role="slider"] ~ div, [data-baseweb="slider"] div[role="slider"] ~ div, @@ -3576,7 +3607,7 @@ def main(): # check_cache_status returns a list of tuples, so check if it has any entries has_stored_data = bool(cache_entries) and len(cache_entries) > 0 except Exception as e: - logger.debug(f"Error checking stored data: {str(e)}") + logger.debug(f"Error checking stored data: {e!s}") has_stored_data = False for idx, step_short in enumerate(step_options): @@ -3594,18 +3625,27 @@ def main(): "background-color: rgba(192, 196, 250, 0.1); border: 1px solid #4313C8; color: #4313C8;" ) else: - highlight_style = "background-color: rgba(192, 196, 250, 0.05); border: 1px solid rgba(67, 19, 200, 0.3); color: #718096;" + highlight_style = ( + "background-color: rgba(192, 196, 250, 0.05); " + "border: 1px solid rgba(67, 19, 200, 0.3); color: #718096;" + ) # Add status badge next to Chunking step - always show status_badge = "" if step_short == "Chunk": badge_text = "Stored" if has_stored_data else "New" badge_bg = "rgba(192, 196, 250, 0.3)" if has_stored_data else "rgba(192, 196, 250, 0.15)" - status_badge = f"{badge_text}" + status_badge = ( + f'{badge_text}' + ) st.markdown( f""" -
+
{indicator} {step_full}{status_badge}
""", @@ -3620,7 +3660,7 @@ def main(): adv_col1, adv_col2 = st.columns(2) with adv_col1: - new_top_k = st.number_input( + st.number_input( "Top K", min_value=1, max_value=20, @@ -3629,7 +3669,7 @@ def main(): on_change=update_analyzer_parameters, ) - new_chunk_size = st.number_input( + st.number_input( "Chunk Size", min_value=100, max_value=2000, @@ -3638,7 +3678,7 @@ def main(): on_change=update_analyzer_parameters, ) - new_overlap = st.number_input( + st.number_input( "Overlap", min_value=0, max_value=100, @@ -3654,7 +3694,7 @@ def main(): selected_index = ( available_llm_models.index(current_model) if current_model in available_llm_models else 0 ) - new_llm_model = st.selectbox( + st.selectbox( "LLM Model", options=available_llm_models, index=selected_index, @@ -3669,14 +3709,14 @@ def main(): disabled=True, ) - new_llm_scoring = st.checkbox( + st.checkbox( "LLM Scoring", value=False, key="new_llm_scoring", on_change=update_analyzer_parameters, ) - new_batch_scoring = st.checkbox( + st.checkbox( "Batch Scoring", value=True, key="new_batch_scoring", @@ -3872,7 +3912,7 @@ def main(): if analyze_clicked or reanalyze_clicked: # NOW sync the selection state from the widget # Get selected questions from the edited dataframe - selected_questions = edited_df[edited_df["Select"] == True]["QID"].tolist() + selected_questions = edited_df[edited_df["Select"] is True]["QID"].tolist() # Update session state for individual question checkboxes (for backward compatibility) for q_id in questions.keys(): @@ -3968,7 +4008,7 @@ def main(): progress_text.success("Analysis complete!") except Exception as e: - st.error(f"Error during analysis: {str(e)}") + st.error(f"Error during analysis: {e!s}") st.exception(e) # Get final results @@ -3989,10 +4029,10 @@ def main(): except Exception as e: logger.error( - f"Error during analysis: {str(e)}", + f"Error during analysis: {e!s}", exc_info=True, ) - st.error(f"Error during analysis: {str(e)}") + st.error(f"Error during analysis: {e!s}") else: # Show helpful error message if file_path is None: @@ -4067,8 +4107,11 @@ def main():
cloud_upload
-

Upload your Sustainability Report

-

Drag and drop your file here, or click to browse.
PDF only, limited to 200MB

+

Upload your Sustainability Report

+

Drag and drop your file here, or click to browse. +
PDF only, limited to 200MB

""", unsafe_allow_html=True, @@ -4265,7 +4308,7 @@ def main(): f"[ENTERPRISE] Error in S3+NATS upload: {e}", exc_info=True, ) - st.error(f"Error uploading via S3+NATS: {str(e)}") + st.error(f"Error uploading via S3+NATS: {e!s}") st.info("Falling back to local processing...") # Fall through to local processing use_s3_upload = False @@ -4296,7 +4339,7 @@ def main(): # Initialize selected_set from session state if available if "consolidated_set" not in st.session_state: - st.session_state.consolidated_set = list(question_sets.keys())[0] if question_sets else None + st.session_state.consolidated_set = next(iter(question_sets.keys())) if question_sets else None # 1. Question set and report selectors side by side (green containers) col1, col2 = st.columns([1, 1]) @@ -4455,7 +4498,10 @@ def main(): # Create clickable card clicked = card( title=model_display, - text=f"Chunk: {config['chunk_size']} · Overlap: {config['chunk_overlap']} · Top-K: {config['top_k']}", + text=( + f"Chunk: {config['chunk_size']} · Overlap: {config['chunk_overlap']} · " + f"Top-K: {config['top_k']}" + ), key=f"config_card_{idx}", styles={ "card": { @@ -4510,6 +4556,42 @@ def main(): selected_config["config"], ) + # Benchmarking page + elif nav_page == "Benchmarking": + st.header("Benchmarking") + st.write("Evaluate retrieval and extraction systems against reference datasets") + + try: + from report_analyst.ui.benchmarking import BenchmarkingUI + + # Initialize analyzer if not already in session state + if "analyzer" not in st.session_state: + st.session_state.analyzer = ReportAnalyzer() + analyzer = st.session_state.analyzer + + benchmark_ui = BenchmarkingUI(analyzer.cache_manager) + + # Sub-tabs for benchmarking features + dataset_tab, eval_tab, results_tab, annotation_tab = st.tabs(["Datasets", "Evaluate", "Results", "Annotate"]) + + with dataset_tab: + benchmark_ui.render_dataset_management() + + with eval_tab: + benchmark_ui.render_benchmarking_interface() + + with results_tab: + benchmark_ui.render_results_dashboard() + + with annotation_tab: + benchmark_ui.render_annotation_interface() + + except ImportError as e: + st.error(f"Benchmarking functionality not available: {e}") + st.exception(e) + except Exception as e: + st.error(f"Error loading benchmarking interface: {e}") + # Add Climate+Tech footer at the bottom of sidebar # Get current theme for logo selection and encode image as base64 try: @@ -4529,16 +4611,24 @@ def main(): # Fallback if logo file doesn't exist logo_src = "" except Exception as e: - logger.warning(f"Could not load logo: {str(e)}") + logger.warning(f"Could not load logo: {e!s}") logo_src = "" # Add footer to sidebar st.sidebar.markdown("---") + logo_img = "" + if logo_src: + logo_img = ( + f'Climate+Tech Logo' + ) footer = f"""
- {f'Climate+Tech Logo' if logo_src else ''} + {logo_img}

Climate+Tech Sustainability Report Analysis Tool

-

For custom tool development contact us at www.climateandtech.com

+

For custom tool development contact us at + www.climateandtech.com

""" st.sidebar.markdown(footer, unsafe_allow_html=True) diff --git a/report_analyst/ui/__init__.py b/report_analyst/ui/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/report_analyst/ui/benchmarking.py b/report_analyst/ui/benchmarking.py new file mode 100644 index 00000000..47a737bf --- /dev/null +++ b/report_analyst/ui/benchmarking.py @@ -0,0 +1,1892 @@ +import logging +import sqlite3 +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pandas as pd +import plotly.express as px +import streamlit as st + +from ..core.benchmark.classification_calibration import ( + compute_calibration_metrics, + compute_classification_report, +) +from ..core.benchmark.dataset_loader import DatasetLoader, DatasetValidationError +from ..core.benchmark.error_analysis import ( + build_error_analysis_dataframe_from_flexible, +) +from ..core.benchmark.evaluation_engine import EvaluationEngine +from ..core.benchmark.flexible_alignment import ( + BenchmarkAlignConfig, + GroundTruthAlignConfig, + align_benchmark_flexible, + align_ground_truth_flexible, +) +from ..core.benchmark.retrieval_results_loader import ( + load_flexible_dataset_from_csv, +) +from ..core.storage.benchmark_store import BenchmarkStore +from ..models.benchmark import ( + BenchmarkDataset, + BenchmarkEvaluation, + DatasetType, + FlexibleDatasetRow, + RetrievalConfig, +) + +logger = logging.getLogger(__name__) + + +class BenchmarkingUI: + """Streamlit UI components for benchmarking functionality""" + + def __init__(self, cache_manager): + self.cache_manager = cache_manager + self.dataset_loader = DatasetLoader() + self.evaluation_engine = EvaluationEngine() + self.benchmark_store = BenchmarkStore(cache_manager.db_path) + + def render_dataset_management(self): + """Render dataset management interface""" + st.subheader("Dataset Management") + + # Global evaluation mode: determines how many datasets are needed + eval_mode = st.radio( + "Select evaluation mode:", + options=["Ranking (retrieval)", "Classification"], + index=0, + key="evaluation_mode", + help=( + "Ranking (retrieval) expects separate ground truth and benchmark datasets. " + "Classification expects a single dataset that already contains both labels " + "and model prediction/score columns." + ), + ) + + # Expected file formats (expandable) + with st.expander("Expected file formats (CSV, Excel, YAML, JSON)"): + st.markdown( + "**CSV / Excel (ground truth or benchmark)** — For evaluation, the app expects " + "columns such as: `query_id` (or `question_id`), `chunk_id`, `position` (or `rank`), " + "`score` (or `relevance_score`). Ground truth alignment expects: `document`, `question`, " + "`context` or `relevant`, `relevance_label`. Benchmark alignment expects: `report`, " + "`question`, `paragraph`, and optionally `relevant_text`, `relevant_text_sim`. " + "Column names are case-insensitive; common variants are accepted. If your file does not " + "match, use **Dataset Alignment** (CSV/Excel only) to convert it." + ) + st.markdown( + "**YAML / JSON** — Benchmark content schema: top-level `dataset_id`, `name`, " + "`description`, `version`, `question_set`, and `questions` (array of `question_id`, " + "`question_text`, `ground_truth_chunks` with `chunk_id`, `relevance_score`, `is_evidence`). " + "Alignment is not available for YAML/JSON; use CSV/Excel for alignment." + ) + st.caption("Full details: see EXPECTED_FILE_FORMATS.md in the project root.") + + # Initialize session state for uploaded datasets if not exists + if "uploaded_datasets" not in st.session_state: + st.session_state.uploaded_datasets = {} + + # Dataset upload section + if eval_mode == "Classification": + # In classification mode we conceptually work with a single dataset + # that already contains both ground-truth labels and model + # predictions/scores. Upload and alignment are handled exclusively + # via the **Flexible Dataset Alignment (Wizard)** below, so we only + # explain the workflow here instead of adding another uploader. + st.subheader("Upload Classification Dataset") + st.write( + "For classification mode, upload a **single CSV/Excel file** that " + "contains both ground-truth label columns (e.g. relevance/usefulness) " + "and one or more prediction/score columns from your models. " + "Use the **Flexible Dataset Alignment (Wizard)** section below to " + "align this file and register it for evaluation." + ) + else: + # For ranking / retrieval mode, all uploads and alignment are done + # via the Flexible Dataset Alignment (Wizard) section below. + st.subheader("Upload Ranking Datasets") + st.write( + "For ranking mode, use the **Flexible Dataset Alignment (Wizard)** " + "section below to upload and align your ground-truth and benchmark " + "CSV/Excel files. Direct YAML/JSON uploads are no longer supported " + "for new evaluations; convert them to CSV/Excel first if needed." + ) + + # Offer download of aligned datasets (when user previously aligned and used one) + aligned_gt = st.session_state.get("aligned_csv_current_ground_truth") + aligned_bm = st.session_state.get("aligned_csv_current_benchmark") + if aligned_gt or aligned_bm: + st.subheader("Download aligned datasets") + st.caption("You have aligned dataset(s) in use. Download the aligned CSV version below.") + dl_col1, dl_col2 = st.columns(2) + if aligned_gt: + with dl_col1: + _name, _bytes = aligned_gt + st.download_button( + label="Download aligned ground truth CSV", + data=_bytes, + file_name="ground_truth_aligned.csv", + mime="text/csv", + key="download_aligned_gt_current", + ) + if aligned_bm: + with dl_col2: + _name, _bytes = aligned_bm + st.download_button( + label="Download aligned benchmark CSV", + data=_bytes, + file_name="benchmark_aligned.csv", + mime="text/csv", + key="download_aligned_bm_current", + ) + + # List existing datasets (optional; alignment should still be available even if empty) + st.subheader("Existing Datasets") + try: + datasets = self.benchmark_store.list_datasets() + except sqlite3.OperationalError as e: + st.warning("Database not initialized. Please restart the application.") + logger.error(f"Database error: {e}") + datasets = [] + + if not datasets: + st.info( + "No datasets have been saved to the database yet. " + "You can still upload CSV/Excel files above and/or use Dataset Alignment below." + ) + else: + # Display datasets in a table + dataset_data = [] + for dataset in datasets: + dataset_data.append( + { + "Dataset ID": dataset.dataset_id, + "Name": dataset.name, + "Question Set": dataset.question_set, + "Version": dataset.version, + "Created": (dataset.created_at.strftime("%Y-%m-%d %H:%M") if dataset.created_at else "Unknown"), + } + ) + + df = pd.DataFrame(dataset_data) + st.dataframe(df, use_container_width=True) + + # Dataset actions + selected_dataset = st.selectbox( + "Select dataset for actions:", + options=[d.dataset_id for d in datasets], + format_func=lambda x: next(d.name for d in datasets if d.dataset_id == x), + ) + + col1, col2 = st.columns(2) + with col1: + if st.button("View Details"): + self._show_dataset_details(selected_dataset) + + with col2: + if st.button("Delete Dataset", type="secondary"): + self._delete_dataset(selected_dataset) + + # ------------------------------------------------------------------ + # Flexible alignment wizard (experimental) + # ------------------------------------------------------------------ + st.subheader("Flexible Dataset Alignment (Wizard)") + if eval_mode == "Classification": + st.caption( + "Use this wizard to upload **one CSV/Excel classification dataset** " + "that already contains both expert labels (e.g. relevance/usefulness) " + "and model prediction/score columns. The aligned dataset will be " + "available for the classification calibration panel." + ) + # For classification we only need a single (results-style) dataset, + # so we reuse the benchmark wizard. + self._render_flexible_bm_wizard() + else: + st.caption( + "Use this wizard when you have custom CSV/Excel files with expert labels " + "and model predictions. It helps you define query IDs, chunk IDs, and " + "label/prediction columns in a consistent way for both ranking and " + "classification evaluation." + ) + + gt_wizard, bm_wizard = st.tabs(["Align Ground Truth (Wizard)", "Align Benchmark (Wizard)"]) + + with gt_wizard: + self._render_flexible_gt_wizard() + + with bm_wizard: + self._render_flexible_bm_wizard() + + def render_benchmarking_interface(self): + """Render benchmarking interface""" + st.subheader("Run Benchmark Evaluation") + + # Get datasets from both database and session state (uploaded files) + db_datasets = [] + try: + db_datasets = self.benchmark_store.list_datasets() + except sqlite3.OperationalError: + # DB may not be initialized yet on first Benchmarking visit. + db_datasets = [] + + uploaded_datasets = st.session_state.get("uploaded_datasets", {}) + + # Check for unconfirmed datasets and show helpful message + temp_datasets = {k: v for k, v in uploaded_datasets.items() if k.startswith("temp_")} + if temp_datasets and not db_datasets and not any(not k.startswith("temp_") for k in uploaded_datasets.keys()): + st.warning( + "⚠️ You have uploaded datasets but haven't confirmed them yet. " + "Please go to the 'Datasets' tab and click 'Confirm and Use This Dataset' " + "for each uploaded dataset to make them available for evaluation." + ) + return + + if not db_datasets and not uploaded_datasets: + st.warning("No datasets available. Please upload datasets first.") + return + + # Choose evaluation type: ranking, classification, or both + eval_mode = st.radio( + "Evaluation type", + options=["Ranking (retrieval)", "Classification", "Both"], + index=0, + key="evaluation_type", + ) + + # Ranking / retrieval evaluation configuration + if eval_mode in ("Ranking (retrieval)", "Both"): + st.subheader("Ranking evaluation (retrieval)") + + col1, col2 = st.columns(2) + + with col1: + # Select reference/ground truth dataset + reference_options = {} + for d in db_datasets: + reference_options[f"DB: {d.name}"] = ("db", d.dataset_id) + + for key, dataset in uploaded_datasets.items(): + # Only show confirmed datasets (not temporary ones) + if "ground_truth" in key and not key.startswith("temp_"): + reference_options[f"Uploaded: {dataset.name}"] = ( + "uploaded", + key, + ) + + if reference_options: + selected_ref_label = st.selectbox( + "Select Reference (Ground Truth) Dataset:", + options=list(reference_options.keys()), + ) + ref_source, ref_id = reference_options[selected_ref_label] + else: + st.error("No reference datasets available") + return + + with col2: + # Select benchmark dataset + benchmark_options = {} + for d in db_datasets: + benchmark_options[f"DB: {d.name}"] = ("db", d.dataset_id) + for key, dataset in uploaded_datasets.items(): + # Only show confirmed datasets (not temporary ones) + if "benchmark" in key and not key.startswith("temp_"): + benchmark_options[f"Uploaded: {dataset.name}"] = ( + "uploaded", + key, + ) + + if benchmark_options: + selected_bench_label = st.selectbox( + "Select Benchmark (Results) Dataset:", + options=list(benchmark_options.keys()), + ) + bench_source, bench_id = benchmark_options[selected_bench_label] + else: + st.error("No benchmark datasets available") + return + + # Evaluation name + evaluation_name = st.text_input( + "Evaluation Name:", + value=f"eval_{pd.Timestamp.now().strftime('%Y%m%d_%H%M')}", + ) + + # Top K configuration + st.subheader("Ranking configuration") + col1, col2 = st.columns(2) + with col1: + st.number_input( + "Top K", + min_value=1, + max_value=50, + value=10, + help="Number of top results to consider for evaluation", + ) + with col2: + k_values_input = st.text_input( + "K values for metrics (comma-separated)", + value="1,3,5,10", + help="K values for Precision@K, Recall@K, etc. (e.g., 1,3,5,10)", + ) + + # Parse K values + k_values = None + if k_values_input: + try: + k_values = [int(k.strip()) for k in k_values_input.split(",") if k.strip()] + if not k_values: + k_values = None + except ValueError: + st.warning("Invalid K values format. Using default values.") + k_values = None + + # Run ranking evaluation + if st.button("Run Ranking Evaluation", type="primary"): + if evaluation_name: + self._run_csv_evaluation_from_datasets( + ref_source, + ref_id, + bench_source, + bench_id, + k_values, + evaluation_name, + ) + else: + st.error("Please provide an evaluation name") + + # Classification evaluation configuration + if eval_mode in ("Classification", "Both"): + st.subheader("Classification evaluation (per-chunk labels and scores)") + st.caption( + "Use this section when your dataset has ground-truth labels (e.g. relevance/usefulness) " + "and one or more numeric prediction/score columns." + ) + # Reuse the classification calibration panel so you can configure and run it here + self._render_classification_calibration_panel("evaluate") + + def render_results_dashboard(self): + """Render evaluation results dashboard""" + st.subheader("Evaluation Results") + + # Get evaluations from database + evaluations = [] + try: + evaluations = self.benchmark_store.list_evaluations() + except sqlite3.OperationalError: + # DB may not be initialized yet; fall back to session evaluations only. + evaluations = [] + + # Get evaluations from session state (recent CSV evaluations) + session_evals = st.session_state.get("csv_evaluations", []) + + all_evaluations = evaluations + session_evals + + if not all_evaluations: + st.info("No evaluations run yet. Run a benchmark evaluation to see results.") + return + + # Filter controls + col1, col2 = st.columns(2) + with col1: + datasets = list(set(e.dataset_id for e in all_evaluations if hasattr(e, "dataset_id"))) + if datasets: + selected_datasets = st.multiselect("Filter by Dataset:", datasets, default=datasets) + else: + selected_datasets = [] + + with col2: + # Show all evaluations by default + pass + + # Filter evaluations + if selected_datasets: + filtered_evals = [e for e in all_evaluations if hasattr(e, "dataset_id") and e.dataset_id in selected_datasets] + else: + filtered_evals = all_evaluations + + if not filtered_evals: + st.warning("No evaluations match the selected filters.") + return + + # Let the user choose which metrics to focus on + metrics_mode = st.radio( + "Select metrics view:", + ["Retrieval", "Classification", "Both"], + index=0, + key="metrics_view_mode", + ) + + if metrics_mode in ("Retrieval", "Both"): + # Results table - show all results + self._render_results_table(filtered_evals) + + # Metrics visualization + self._render_metrics_charts(filtered_evals) + + # Error analysis export for the latest CSV evaluation (session-based) + self._render_error_analysis_export() + + if metrics_mode in ("Classification", "Both"): + # Model comparison from stored classification calibration runs + self._render_classification_model_comparison() + # Classification calibration for datasets with labels/scores + self._render_classification_calibration_panel("result") + + # Detailed evaluation view + if filtered_evals: + eval_options = [f"{e.evaluation_name} ({getattr(e, 'dataset_id', 'N/A')})" for e in filtered_evals] + selected_eval_name = st.selectbox( + "Select evaluation for details:", + options=eval_options, + ) + if selected_eval_name: + selected_idx = eval_options.index(selected_eval_name) + selected_eval = filtered_evals[selected_idx] + self._render_evaluation_details(selected_eval) + + def _render_error_analysis_export(self): + """Render download button for error-analysis CSV based on current CSV evaluation context.""" + # Use the most recent CSV evaluation from session state + session_evals = st.session_state.get("csv_evaluations", []) + if not session_evals: + return + + latest_eval = session_evals[-1] + retrieval_config = getattr(latest_eval, "retrieval_config", None) + if not retrieval_config: + return + + top_k = getattr(retrieval_config, "top_k", None) + if not top_k: + return + + st.subheader("Error analysis export") + st.caption( + "Download a CSV with, for each retrieved chunk, the report, question, expert relevant part, " + "retrieved chunk text, its position in top-K, expert relevance label, and whether it is really relevant " + "(expert label > 0)." + ) + + if st.button("Build error-analysis CSV"): + try: + # Get current ground-truth and benchmark datasets used for the last CSV evaluation + # Prefer explicit keys stored on the evaluation object + ref_id = getattr(latest_eval, "ref_key", None) + bench_id = getattr(latest_eval, "bench_key", None) + + dataset_id = getattr(latest_eval, "dataset_id", "") + # Fallback to parsing dataset_id for legacy evaluations + if (ref_id is None or bench_id is None) and dataset_id: + if "|||" in dataset_id: + ref_id, bench_id = dataset_id.split("|||", 1) + elif "_" in dataset_id: + ref_id, bench_id = dataset_id.split("_", 1) + + if not ref_id or not bench_id: + st.error("Cannot determine datasets for error analysis from evaluation metadata.") + return + + uploaded_datasets = st.session_state.get("uploaded_datasets", {}) + ground_truth_ds = uploaded_datasets.get(ref_id) + benchmark_ds = uploaded_datasets.get(bench_id) + + if not ground_truth_ds or not benchmark_ds: + st.error("Could not locate the ground truth and benchmark datasets used for this evaluation.") + return + + df_error = build_error_analysis_dataframe_from_flexible( + ground_truth_dataset=ground_truth_ds, + benchmark_dataset=benchmark_ds, + top_k=top_k, + ) + + if df_error.empty: + st.warning("No rows generated for error analysis.") + return + + csv_bytes = df_error.to_csv(index=False).encode("utf-8") + st.download_button( + "Download error-analysis CSV", + data=csv_bytes, + file_name="benchmark_error_analysis.csv", + mime="text/csv", + key="download_error_analysis_csv", + ) + except Exception as e: # noqa: BLE001 + st.error(f"Failed to build error-analysis CSV: {e!s}") + + def render_annotation_interface(self): + """Render human annotation interface""" + st.subheader("Human Annotation") + + evaluations = self.benchmark_store.list_evaluations() + if not evaluations: + st.info("No evaluations available for annotation.") + return + + selected_eval = st.selectbox( + "Select Evaluation to Annotate:", + options=evaluations, + format_func=lambda x: f"{x.evaluation_name} ({x.dataset_id})", + ) + + if selected_eval: + self._render_annotation_form(selected_eval) + + def _render_classification_model_comparison(self): + """Render stored classification calibration runs as model comparison table and charts.""" + runs = st.session_state.get("csv_classification_evaluations", []) + if not runs: + st.info( + "Run classification calibration in the **Evaluate** tab (select multiple " + "prediction/score columns) to compare models here." + ) + return + + st.subheader("Classification model comparison") + options = [f"{r['evaluation_name']} ({r['created_at'].strftime('%Y-%m-%d %H:%M')})" for r in runs] + selected_idx = st.selectbox( + "Select calibration run:", + range(len(runs)), + format_func=lambda i: options[i], + key="results_classification_run_select", + ) + run = runs[selected_idx] + metrics_df = run["metrics_df"] + if metrics_df.empty: + st.warning("No metrics for this run.") + return + + st.caption(f"Run: **{run['evaluation_name']}** — one row per model (prediction column).") + st.dataframe(metrics_df, use_container_width=True) + + def _render_classification_calibration_panel(self, key_prefix: str): + """Render classification calibration metrics for datasets with relevance/usefulness scores.""" + uploaded_datasets = st.session_state.get("uploaded_datasets", {}) + if not uploaded_datasets: + return + + # Build list of datasets that have at least some numeric columns + candidates = [] + for key, dataset in uploaded_datasets.items(): + results = getattr(dataset, "results", []) + if not results: + continue + first = results[0] + data = getattr(first, "data", {}) or {} + if not data: + continue + df_sample = pd.DataFrame([data]) + numeric_cols = df_sample.select_dtypes(include=["number"]).columns.tolist() + if numeric_cols: + candidates.append((key, dataset)) + + if not candidates: + return + + st.subheader("Classification calibration") + st.caption( + "Analyze calibration and classification performance for any dataset with " + "a label column and one or more numeric prediction/score columns." + ) + + # Dataset selector + labels = [f"{ds.name} ({key})" for key, ds in candidates] + selected_label = st.selectbox( + "Select dataset for calibration analysis:", + options=labels, + key=f"{key_prefix}_calibration_dataset_select", + ) + idx = labels.index(selected_label) + selected_key, selected_dataset = candidates[idx] + + # Build DataFrame from FlexibleDatasetRow objects + df_rows = [row.data for row in selected_dataset.results] + df = pd.DataFrame(df_rows) + if df.empty: + st.info("Selected dataset has no rows to analyze.") + return + + all_cols = df.columns.tolist() + + # Try to load defaults from dataset metadata (set by the flexible wizard + # when running in classification mode). Falls back to heuristics if not set. + mapping = getattr(selected_dataset, "column_mapping", {}) or {} + default_label_name = mapping.get("classification_label_col") + raw_pred_cols = mapping.get("classification_prediction_cols") or "" + # Stored as a comma-separated string to keep column_mapping values simple. + default_pred_cols = ( + [c.strip() for c in raw_pred_cols.split(",") if c.strip()] if isinstance(raw_pred_cols, str) else [] + ) + + # Ground-truth label: let the user pick from **all** columns. + label_candidates = all_cols + + if default_label_name and default_label_name in label_candidates: + default_label_index = label_candidates.index(default_label_name) + else: + # Prefer relevance/usefulness as default if present; otherwise first column. + default_label_index = 0 + for preferred in ("relevance", "usefulness"): + for i, col in enumerate(label_candidates): + if col.lower() == preferred: + default_label_index = i + break + else: + continue + break + + label_col = st.selectbox( + "Ground-truth label column (e.g. relevance or usefulness):", + options=label_candidates, + index=default_label_index, + key=f"{key_prefix}calibration_label_select", + ) + + # Prediction / score columns: show **all** columns and let the user decide. + score_candidates = all_cols + + # Default predictions: use stored configuration if available; otherwise none. + default_score_selection = [c for c in default_pred_cols if c in score_candidates] + + # Let the user choose one or more prediction columns (models) to analyze. + selected_score_cols = st.multiselect( + "Prediction / score columns (models):", + options=score_candidates, + default=default_score_selection, + key=f"{key_prefix}calibration_score_multiselect", + ) + if not selected_score_cols: + st.info("Select at least one prediction/score column to compute metrics.") + return + + n_bins = st.slider( + "Number of bins for Expected Calibration Error (ECE):", + min_value=10, + max_value=200, + value=100, + step=10, + key=f"{key_prefix}calibration_ece_bins", + ) + + if st.button("Compute classification calibration", key=f"{key_prefix}calibration_compute"): + with st.spinner("Computing calibration metrics..."): + try: + metrics_df = compute_calibration_metrics( + df, + ground_truth_col=label_col, + score_cols=selected_score_cols, + n_bins=n_bins, + ) + except Exception as exc: + st.error(f"Failed to compute calibration metrics: {exc}") + logger.exception("Calibration metrics error") + return + + st.markdown("**Calibration and discrimination metrics**") + st.dataframe(metrics_df, use_container_width=True) + + # Store for model comparison in Results tab + if "csv_classification_evaluations" not in st.session_state: + st.session_state.csv_classification_evaluations = [] + eval_name = f"Classification: {selected_key} ({label_col})" + st.session_state.csv_classification_evaluations.append( + { + "evaluation_name": eval_name, + "dataset_id": selected_key, + "label_col": label_col, + "metrics_df": metrics_df, + "created_at": pd.Timestamp.now(), + } + ) + st.success("Saved for model comparison in the **Results** tab.") + + # Classification report for every selected model + for score_col in selected_score_cols: + try: + report_dict = compute_classification_report( + df, + ground_truth_col=label_col, + score_col=score_col, + ) + except Exception as exc: + st.error(f"Failed to compute classification report for `{score_col}`: {exc}") + logger.exception("Classification report error") + continue + + if report_dict: + st.markdown(f"**Classification report for `{score_col}` (labels 0, 1, 2)**") + report_df = pd.DataFrame(report_dict).T + st.dataframe(report_df, use_container_width=True) + else: + st.info(f"Not enough non-missing data to compute a classification report for `{score_col}`.") + + def _run_csv_evaluation_from_datasets(self, ref_source, ref_id, bench_source, bench_id, k_values, evaluation_name): + """Run evaluation using selected datasets from database or uploaded files""" + try: + with st.spinner("Loading datasets and calculating metrics..."): + # Load reference dataset + if ref_source == "db": + # Load from database (would need to convert from BenchmarkDatasetContent) + st.error("Database dataset evaluation not yet implemented. Please use uploaded CSV/Excel files.") + return + else: + # Load from session state + reference_dataset = st.session_state.uploaded_datasets.get(ref_id) + if not reference_dataset: + st.error(f"Reference dataset {ref_id} not found.") + return + + # Load benchmark dataset + if bench_source == "db": + st.error("Database dataset evaluation not yet implemented. Please use uploaded CSV/Excel files.") + return + else: + # Load from session state + benchmark_dataset = st.session_state.uploaded_datasets.get(bench_id) + if not benchmark_dataset: + st.error(f"Benchmark dataset {bench_id} not found.") + return + + # Run evaluation + metrics = self.evaluation_engine.compare_flexible_datasets( + reference_dataset, benchmark_dataset, k_values=k_values + ) + + # Display results + st.success("Evaluation completed successfully!") + st.subheader("Results") + + # Display metrics table + self._render_csv_metrics_table(metrics, k_values) + + # Display charts + self._render_csv_metrics_charts(metrics, k_values) + + # Store evaluation in session state for results tab + if "csv_evaluations" not in st.session_state: + st.session_state.csv_evaluations = [] + + from ..models.benchmark import RetrievalConfig + + # Use the maximum k value for error analysis export (to show all top-K chunks) + max_k = max(k_values) if k_values else 10 + eval_obj = type( + "Evaluation", + (), + { + "evaluation_name": evaluation_name, + "dataset_id": f"{ref_id}|||{bench_id}", + "evaluation_metrics": metrics, + "retrieval_config": RetrievalConfig(top_k=max_k), + "created_at": pd.Timestamp.now(), + # Explicit references to the dataset keys used for this evaluation + "ref_key": ref_id, + "bench_key": bench_id, + }, + )() + st.session_state.csv_evaluations.append(eval_obj) + + except ValueError as e: + st.error(f"Error loading datasets: {e!s}") + logger.exception("Dataset loading error") + except Exception as e: + st.error(f"Error during evaluation: {e!s}") + logger.exception("Evaluation error") + + def _render_confirmation_ui(self, dataset, temp_key: str, dataset_type: str, file_name: str): + """Render confirmation UI for an unconfirmed dataset""" + st.info(f"📋 **{dataset.name}** ({len(dataset.results)} rows) - Pending confirmation") + + # Show basic info + st.write(f"**Dataset Name:** {dataset.name}") + st.write(f"**Number of rows:** {len(dataset.results)}") + + # Confirm save - this will replace any existing dataset of the same type + col1, col2 = st.columns(2) + with col1: + button_key = f"confirm_{dataset_type}_{file_name}" + if st.button("Confirm and Use This Dataset", key=button_key, type="primary"): + dataset_key = f"{dataset_type}_current" + + # Remove old dataset of the same type if it exists + if dataset_key in st.session_state.uploaded_datasets: + old_dataset = st.session_state.uploaded_datasets[dataset_key] + st.info(f"Replacing previous {dataset_type} dataset: {old_dataset.name}") + + # Save the new dataset + st.session_state.uploaded_datasets[dataset_key] = dataset + + # Remove temporary key + if temp_key in st.session_state.uploaded_datasets: + del st.session_state.uploaded_datasets[temp_key] + + st.success(f"Dataset '{dataset.name}' confirmed and ready for evaluation!") + st.rerun() + + with col2: + if st.button("Cancel", key=f"cancel_{dataset_type}_{file_name}"): + # Remove temporary dataset + if temp_key in st.session_state.uploaded_datasets: + del st.session_state.uploaded_datasets[temp_key] + st.info("Upload cancelled. Dataset not saved.") + st.rerun() + + # Show detailed preview + st.divider() + st.write("**Dataset Details:**") + st.write(f"**Dataset ID:** {dataset.dataset_id}") + st.write(f"**Dataset Type:** {dataset.dataset_type.value}") + + # Show sample data + if dataset.results: + st.write("**Sample data (first 5 rows):**") + sample_data = [] + for _i, result in enumerate(dataset.results[:5]): + row_data = result.data.copy() + sample_data.append(row_data) + if sample_data: + st.dataframe(pd.DataFrame(sample_data), use_container_width=True) + + def _handle_dataset_upload(self, uploaded_file, dataset_type: str = "ground_truth"): + """Handle dataset file upload - supports YAML, JSON, CSV, and Excel files""" + tmp_path = None + try: + # Determine file extension + file_ext = uploaded_file.name.split(".")[-1].lower() + + # Save uploaded file temporarily + with tempfile.NamedTemporaryFile(delete=False, suffix=f".{file_ext}") as tmp_file: + tmp_file.write(uploaded_file.getvalue()) + tmp_path = tmp_file.name + + # Load dataset based on file type + with st.spinner("Loading and validating dataset..."): + if file_ext in ["csv", "xlsx", "xls"]: + # ------------------------------------------------------------------ + # Step 1: Read raw tabular data + # ------------------------------------------------------------------ + if file_ext in ["xlsx", "xls"]: + df_raw = pd.read_excel(tmp_path) + csv_string = df_raw.to_csv(index=False) + csv_kwargs = {"csv_content": csv_string} + else: + df_raw = pd.read_csv(tmp_path) + csv_kwargs = {"csv_path": tmp_path} + + # ------------------------------------------------------------------ + # Special path: classification mode. + # In this mode we want to preserve all label/prediction columns and + # only add IDs (query_id, chunk_id) via classification alignment, + # without forcing the strict retrieval schema. + # ------------------------------------------------------------------ + if dataset_type == "classification": + self._handle_classification_upload( + df_raw=df_raw, + uploaded_file=uploaded_file, + dataset_type=dataset_type, + ) + return + + # If this file has already been aligned and registered for this + # dataset_type, avoid re-showing the schema warning on rerun. + aligned_flag_key = f"aligned_{dataset_type}_{uploaded_file.name}" + dataset_key = f"{dataset_type}_current" + if ( + st.session_state.get(aligned_flag_key) + and "uploaded_datasets" in st.session_state + and dataset_key in st.session_state.uploaded_datasets + ): + aligned_dataset = st.session_state.uploaded_datasets[dataset_key] + st.success( + f"Using previously aligned dataset '{aligned_dataset.name}' " + f"({len(aligned_dataset.results)} rows) for evaluation." + ) + return + + # ------------------------------------------------------------------ + # Step 2: Try to load as a flexible benchmark dataset + # If this fails (missing key columns, etc.), offer alignment. + # ------------------------------------------------------------------ + dataset = None + load_error: Optional[Exception] = None + try: + dataset = load_flexible_dataset_from_csv( + dataset_name=f"{dataset_type}_{uploaded_file.name}", + **csv_kwargs, + ) + except Exception as exc: # noqa: BLE001 # pragma: no cover - UI fallback + load_error = exc + + if dataset is not None: + st.success(f"Dataset loaded successfully from {file_ext.upper()} file!") + + # Store dataset temporarily in session state for preview + # Use a temporary key that will be replaced on confirmation + temp_key = f"temp_{dataset_type}_{uploaded_file.name}" + if "uploaded_datasets" not in st.session_state: + st.session_state.uploaded_datasets = {} + + st.session_state.uploaded_datasets[temp_key] = dataset + + # Show basic info and confirmation buttons immediately after upload + st.write(f"**Dataset Name:** {dataset.name}") + st.write(f"**Number of rows:** {len(dataset.results)}") + + # Confirm save - this will replace any existing dataset of the same type + col1, col2 = st.columns(2) + with col1: + button_key = f"confirm_{dataset_type}_{uploaded_file.name}" + if st.button( + "Confirm and Use This Dataset", + key=button_key, + type="primary", + ): + dataset_key = f"{dataset_type}_current" + + # Remove old dataset of the same type if it exists + if dataset_key in st.session_state.uploaded_datasets: + old_dataset = st.session_state.uploaded_datasets[dataset_key] + st.info(f"Replacing previous {dataset_type} dataset: {old_dataset.name}") + + # Save the new dataset + st.session_state.uploaded_datasets[dataset_key] = dataset + + # Remove temporary key + if temp_key in st.session_state.uploaded_datasets: + del st.session_state.uploaded_datasets[temp_key] + + # Clear any previously stored aligned CSV for this type + st.session_state.pop(f"aligned_csv_current_{dataset_type}", None) + + st.success(f"Dataset '{dataset.name}' confirmed and ready for evaluation!") + st.rerun() + + with col2: + if st.button( + "Cancel", + key=f"cancel_{dataset_type}_{uploaded_file.name}", + ): + # Remove temporary dataset + if temp_key in st.session_state.uploaded_datasets: + del st.session_state.uploaded_datasets[temp_key] + st.info("Upload cancelled. Dataset not saved.") + st.rerun() + + # Show detailed preview below the buttons + st.divider() + st.write("**Dataset Details:**") + st.write(f"**Dataset ID:** {dataset.dataset_id}") + st.write(f"**Dataset Type:** {dataset.dataset_type.value}") + + if dataset.results: + st.write("**Sample data (first 5 rows):**") + sample_data = [] + for _i, result in enumerate(dataset.results[:5]): + row_data = result.data.copy() + sample_data.append(row_data) + if sample_data: + st.dataframe( + pd.DataFrame(sample_data), + use_container_width=True, + ) + else: + # ------------------------------------------------------------------ + # Dataset is not in the expected benchmark schema. + # Offer alignment using DatasetMapper. + # ------------------------------------------------------------------ + st.warning("This file does not match the expected benchmark CSV schema used by the evaluation engine.") + st.caption( + "For new or custom CSV/Excel files, please use the " + "**Flexible Dataset Alignment (Wizard)** section below to " + "map your columns to the internal schema. The older " + "preset/manual alignment UI has been removed." + ) + if load_error is not None: + st.caption(f"Details: {load_error}") + return + else: + # Use traditional YAML/JSON loader + dataset = self.dataset_loader.load_dataset(tmp_path) + warnings = self.dataset_loader.validate_dataset_consistency(dataset) + + # Show validation results + if warnings: + st.warning(f"Dataset loaded with {len(warnings)} warnings:") + for warning in warnings: + st.write(f"Warning: {warning}") + else: + st.success("Dataset validation passed!") + + # Show dataset preview + st.write(f"**Dataset:** {dataset.name}") + st.write(f"**Description:** {dataset.description}") + st.write(f"**Questions:** {len(dataset.questions)}") + + # Confirm save + if st.button("Save Dataset", key=f"save_{dataset_type}"): + self.benchmark_store.save_dataset(dataset, uploaded_file.name) + st.success(f"Dataset '{dataset.name}' saved successfully!") + st.rerun() + + except (DatasetValidationError, ValueError) as e: + st.error(f"Failed to load dataset: {e!s}") + logger.exception("Dataset loading error") + except Exception as e: + st.error(f"Unexpected error: {e!s}") + logger.exception("Unexpected error during dataset upload") + + finally: + # Clean up temp file + if tmp_path: + try: + Path(tmp_path).unlink() + except OSError as exc: + logger.debug("Temp file cleanup failed: %s", exc) + + def _handle_classification_upload(self, df_raw: pd.DataFrame, uploaded_file, dataset_type: str): + """Handle upload of a classification-style dataset (single file with labels and predictions). + + In this simplified path we do **no special alignment** - we just wrap the + raw DataFrame rows as `FlexibleDatasetRow` objects so that the calibration + panel can let the user pick any label and prediction columns manually. + """ + if df_raw.empty: + st.error("Uploaded classification dataset is empty.") + return + + # Build BenchmarkDataset directly from the raw DataFrame, preserving all columns + results = [] + for _, row in df_raw.iterrows(): + row_dict = {} + for col in df_raw.columns: + val = row[col] + row_dict[col] = None if pd.isna(val) else val + results.append(FlexibleDatasetRow(data=row_dict)) + + dataset = BenchmarkDataset( + dataset_id=f"{dataset_type}_{uploaded_file.name}", + name=uploaded_file.name, + description="Classification dataset (raw; no alignment)", + version="1.0", + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + source="csv", + source_path=None, + column_mapping={}, + results=results, + ) + + if "uploaded_datasets" not in st.session_state: + st.session_state.uploaded_datasets = {} + + dataset_key = f"{dataset_type}_current" + st.session_state.uploaded_datasets[dataset_key] = dataset + + # Store a copy of the original CSV for download convenience + csv_bytes = df_raw.to_csv(index=False).encode("utf-8") + st.session_state[f"aligned_csv_current_{dataset_type}"] = ( + uploaded_file.name, + csv_bytes, + ) + + st.success( + f"Classification dataset '{uploaded_file.name}' loaded and ready for " + "manual column selection in the classification calibration panel." + ) + + def _render_flexible_gt_wizard(self): + """Guided wizard for aligning a ground truth dataset to the flexible schema.""" + gt_file = st.file_uploader( + "Ground truth CSV/Excel for flexible alignment", + type=["csv", "xlsx", "xls"], + key="flex_gt_file", + ) + if gt_file is None: + return + + ext = gt_file.name.split(".")[-1].lower() + if ext in ["xlsx", "xls"]: + df_raw = pd.read_excel(gt_file) + else: + df_raw = pd.read_csv(gt_file) + + if df_raw.empty: + st.warning("Uploaded ground truth file is empty.") + return + + st.write("Preview of uploaded ground truth data:") + st.dataframe(df_raw.head(), use_container_width=True) + + cols = list(df_raw.columns) + cols_lower = {c.lower(): c for c in cols} + + # Query definition + st.markdown("**Step 1 - Identify the query (report + question)**") + doc_options = ["", *cols] + default_doc_idx = 0 + for cand in ("document", "report", "report_name", "doc_id"): + if cand in cols_lower: + default_doc_idx = doc_options.index(cols_lower[cand]) + break + document_sel = st.selectbox( + "Report / company column (used to build query ID):", + options=doc_options, + index=default_doc_idx, + key="flex_gt_document_col", + ) + document_col = None if document_sel == "" else document_sel + + # Choose question/description column + default_q = cols[0] + for cand in ("description", "question", "criteria", "prompt"): + if cand in cols_lower: + default_q = cols_lower[cand] + break + question_col = st.selectbox( + "Criteria / question text column:", + options=cols, + index=cols.index(default_q) if default_q in cols else 0, + key="flex_gt_question_col", + ) + + default_chunk = cols[0] + for cand in ("chunk_text", "paragraph", "context", "text"): + if cand in cols_lower: + default_chunk = cols_lower[cand] + break + # Chunk & relevant part + st.markdown("**Step 2 - Ground-truth text: full chunk vs. relevant part**") + chunk_text_col = st.selectbox( + "Column with the full chunk / paragraph text (or the expert relevant text if no full chunk is stored):", + options=cols, + index=cols.index(default_chunk) if default_chunk in cols else 0, + key="flex_gt_chunk_col", + ) + + rel_options = ["", *cols] + default_rel_idx = 0 + for cand in ("relevant_part_text", "relevant", "relevant_text"): + if cand in cols_lower: + default_rel_idx = rel_options.index(cols_lower[cand]) + break + rel_sel = st.selectbox( + ( + "Column with the expert-labeled relevant span inside the chunk " + "(optional - if you only have relevant text, you can reuse the " + "same column as above):" + ), + options=rel_options, + index=default_rel_idx, + key="flex_gt_relevant_col", + ) + relevant_part_col = None if rel_sel == "" else rel_sel + + # Label columns + st.markdown("**Step 3 - Ground-truth relevance labels**") + # Suggest numeric or name-based label columns + df_sample = df_raw.head(50) + numeric_cols = df_sample.select_dtypes(include=["number"]).columns.tolist() + label_suggestions = set() + for c in cols: + cl = c.lower() + if any(x in cl for x in ("relevance", "usefulness", "label", "class")): + label_suggestions.add(c) + if not label_suggestions: + label_suggestions.update(numeric_cols) + default_labels = sorted(label_suggestions) + + label_cols = st.multiselect( + "Select label column(s) with expert scores (e.g. relevance, usefulness):", + options=cols, + default=default_labels, + key="flex_gt_label_cols", + ) + + if st.button( + "Align ground truth (flexible) and use for evaluation", + key="flex_gt_align_button", + ): + try: + gt_config = GroundTruthAlignConfig( + document_col=document_col, + question_col=question_col, + chunk_text_col=chunk_text_col, + relevant_part_col=relevant_part_col, + label_cols=label_cols, + ) + df_aligned = align_ground_truth_flexible(df_raw, gt_config) + + st.success( + f"Aligned flexible ground truth: {len(df_aligned)} rows, {df_aligned['query_id'].nunique()} queries." + ) + st.dataframe(df_aligned.head(), use_container_width=True) + + # Build BenchmarkDataset from aligned DataFrame + results: List[FlexibleDatasetRow] = [] + for _, row in df_aligned.iterrows(): + row_dict: Dict[str, Any] = {} + for col in df_aligned.columns: + val = row[col] + row_dict[col] = None if pd.isna(val) else val + results.append(FlexibleDatasetRow(data=row_dict)) + + dataset = BenchmarkDataset( + dataset_id=f"ground_truth_flexible_{gt_file.name}", + name=f"GT flexible ({gt_file.name})", + description="Ground truth dataset aligned via flexible wizard", + version="1.0", + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + source="csv", + source_path=None, + column_mapping={}, + results=results, + ) + + if "uploaded_datasets" not in st.session_state: + st.session_state.uploaded_datasets = {} + + # Use a key that still matches 'ground_truth' for the evaluation UI + dataset_key = "ground_truth_flexible_current" + st.session_state.uploaded_datasets[dataset_key] = dataset + + csv_bytes = df_aligned.to_csv(index=False).encode("utf-8") + st.session_state["aligned_csv_current_ground_truth"] = ( + gt_file.name, + csv_bytes, + ) + + st.success("Flexible ground truth dataset is now registered for evaluation.") + except Exception as exc: # noqa: BLE001 + st.error(f"Flexible GT alignment failed: {exc}") + + def _render_flexible_bm_wizard(self): + """Guided wizard for aligning a benchmark/results dataset to the flexible schema.""" + bm_file = st.file_uploader( + "Benchmark results CSV/Excel for flexible alignment", + type=["csv", "xlsx", "xls"], + key="flex_bm_file", + ) + if bm_file is None: + return + + ext = bm_file.name.split(".")[-1].lower() + if ext in ["xlsx", "xls"]: + df_raw = pd.read_excel(bm_file) + else: + df_raw = pd.read_csv(bm_file) + + if df_raw.empty: + st.warning("Uploaded benchmark file is empty.") + return + + st.write("Preview of uploaded benchmark data:") + st.dataframe(df_raw.head(), use_container_width=True) + + cols = list(df_raw.columns) + cols_lower = {c.lower(): c for c in cols} + + # Query linkage + has_query_id_col = any(c.lower() == "query_id" for c in cols) + eval_mode = st.session_state.get("evaluation_mode") + if eval_mode == "Classification": + # Classification mode: query_id is not used for metrics, but we still + # let the user label report and criteria columns for context and + # for potential grouping in future analyses. + st.markdown("**Step 1 - Report and criteria (optional context)**") + doc_options = ["", *cols] + default_doc_idx = 0 + for cand in ("document", "report", "report_name", "doc_id", "company"): + if cand in cols_lower: + default_doc_idx = doc_options.index(cols_lower[cand]) + break + document_sel = st.selectbox( + "Report / company column (optional):", + options=doc_options, + index=default_doc_idx, + key="flex_bm_document_col", + ) + document_col = None if document_sel == "" else document_sel + + default_q = cols[0] + for cand in ("description", "question", "criteria", "prompt"): + if cand in cols_lower: + default_q = cols_lower[cand] + break + question_col = st.selectbox( + "Criteria / question text column (optional):", + options=cols, + index=cols.index(default_q) if default_q in cols else 0, + key="flex_bm_question_col", + ) + # We don't configure qid_col explicitly; align_benchmark_flexible will + # derive query_id from document/question as needed. + qid_col = None + else: + st.markdown("**Step 1 - Match this file to the ground-truth queries**") + query_id_options = ["", *cols] + default_qid_idx = 0 + if has_query_id_col: + default_qid_idx = query_id_options.index(next(c for c in cols if c.lower() == "query_id")) + query_id_sel = st.selectbox( + "How should the query ID be obtained for this file?", + options=query_id_options, + index=default_qid_idx, + key="flex_bm_query_id_strategy", + ) + if query_id_sel == "": + qid_col = None + # Need document and question/description columns + doc_options = ["", *cols] + default_doc_idx = 0 + for cand in ("document", "report", "report_name", "doc_id"): + if cand in cols_lower: + default_doc_idx = doc_options.index(cols_lower[cand]) + break + document_sel = st.selectbox( + "Report / company column (must match the ground-truth file):", + options=doc_options, + index=default_doc_idx, + key="flex_bm_document_col", + ) + document_col = None if document_sel == "" else document_sel + + default_q = cols[0] + for cand in ("description", "question", "criteria", "prompt"): + if cand in cols_lower: + default_q = cols_lower[cand] + break + question_col = st.selectbox( + "Criteria / question text column (must match the ground-truth file):", + options=cols, + index=cols.index(default_q) if default_q in cols else 0, + key="flex_bm_question_col", + ) + else: + qid_col = query_id_sel + document_col = None + question_col = None + + # Chunk & optional relevant part + st.markdown("**Step 2 - Retrieved chunk and (optional) relevant span**") + default_chunk = cols[0] + for cand in ("chunk_text", "paragraph", "context", "text"): + if cand in cols_lower: + default_chunk = cols_lower[cand] + break + chunk_text_col = st.selectbox( + "Column with the retrieved chunk / paragraph text:", + options=cols, + index=cols.index(default_chunk) if default_chunk in cols else 0, + key="flex_bm_chunk_col", + ) + + rel_options = ["", *cols] + default_rel_idx = 0 + for cand in ("relevant_part_text_pred", "relevant_text_pred", "relevant_text"): + if cand in cols_lower: + default_rel_idx = rel_options.index(cols_lower[cand]) + break + rel_sel = st.selectbox( + ( + "Column with the text span that should match the ground-truth " + "relevant part (for datasets with expert spans like ClimRetrieve " + "this is strongly recommended; leave as only if your " + "ground truth is chunk-level only):" + ), + options=rel_options, + index=default_rel_idx, + key="flex_bm_relevant_col", + ) + relevant_part_col = None if rel_sel == "" else rel_sel + + # Detect if we are in classification mode. In that case, we want the + # wizard to also capture which column is the ground-truth label and + # which columns are model predictions so that the evaluation tab can + # pre-populate the calibration UI. + is_classification_mode = st.session_state.get("evaluation_mode") == "Classification" + + # Prediction & similarity columns + st.markdown("**Step 3 - Model scores and ranking signal**") + df_sample = df_raw.head(50) + numeric_cols = df_sample.select_dtypes(include=["number"]).columns.tolist() + pred_suggestions = set() + for c in cols: + cl = c.lower() + if any( + x in cl + for x in ( + "score", + "sim", + "prob", + "logit", + "confidence", + "relevance", + "usefulness", + ) + ): + pred_suggestions.add(c) + if not pred_suggestions: + pred_suggestions.update(numeric_cols) + + sorted(pred_suggestions) + + classification_label_col = None + if is_classification_mode: + st.markdown("**Step 3a - Ground-truth label used for classification metrics**") + # Heuristic: prefer 'relevance' or 'usefulness' as label. + default_label = cols[0] + for cand in ("relevance", "usefulness", "label", "class"): + for c in cols: + if c.lower() == cand: + default_label = c + break + else: + continue + break + classification_label_col = st.selectbox( + "Column with expert labels (e.g. relevance or usefulness):", + options=cols, + index=cols.index(default_label) if default_label in cols else 0, + key="flex_bm_classification_label_col", + ) + + st.markdown("**Step 3b - Prediction / score columns from your models**") + prediction_cols = st.multiselect( + "Select model prediction / score column(s) (e.g. similarity, relevance_score_*):", + options=cols, + default=[], + key="flex_bm_prediction_cols", + ) + + ranking_score_col = None + if prediction_cols: + ranking_score_col = st.selectbox( + "Which score should be used to rank chunks and decide top-K relevance?", + options=prediction_cols, + index=0, + key="flex_bm_ranking_score_col", + ) + + if st.button( + "Align benchmark (flexible) and use for evaluation", + key="flex_bm_align_button", + ): + try: + bm_config = BenchmarkAlignConfig( + document_col=document_col, + question_col=question_col, + query_id_col=qid_col, + chunk_text_col=chunk_text_col, + relevant_part_col=relevant_part_col, + prediction_cols=prediction_cols, + ranking_score_col=ranking_score_col, + ) + df_aligned = align_benchmark_flexible(df_raw, bm_config) + + # In classification mode, ensure the chosen ground-truth label + # column is preserved in the aligned DataFrame so that the + # evaluation tab can see and use it. + if is_classification_mode and classification_label_col: + if classification_label_col in df_raw.columns: + df_aligned[classification_label_col] = df_raw[classification_label_col].values + + st.success(f"Aligned flexible benchmark: {len(df_aligned)} rows, {df_aligned['query_id'].nunique()} queries.") + st.dataframe(df_aligned.head(), use_container_width=True) + + results: List[FlexibleDatasetRow] = [] + for _, row in df_aligned.iterrows(): + row_dict: Dict[str, Any] = {} + for col in df_aligned.columns: + val = row[col] + row_dict[col] = None if pd.isna(val) else val + results.append(FlexibleDatasetRow(data=row_dict)) + + dataset = BenchmarkDataset( + dataset_id=f"benchmark_flexible_{bm_file.name}", + name=f"BM flexible ({bm_file.name})", + description="Benchmark dataset aligned via flexible wizard", + version="1.0", + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + source="csv", + source_path=None, + column_mapping={ + # Remember classification defaults (if any) so that the + # calibration panel can pre-populate label/prediction + # selections without forcing the user to configure them again. + "classification_label_col": (classification_label_col if is_classification_mode else ""), + # Store prediction columns as a single comma-separated string + # to keep column_mapping values as simple scalars. + "classification_prediction_cols": (",".join(prediction_cols) if is_classification_mode else ""), + }, + results=results, + ) + + if "uploaded_datasets" not in st.session_state: + st.session_state.uploaded_datasets = {} + + dataset_key = "benchmark_flexible_current" + st.session_state.uploaded_datasets[dataset_key] = dataset + + csv_bytes = df_aligned.to_csv(index=False).encode("utf-8") + st.session_state["aligned_csv_current_benchmark"] = ( + bm_file.name, + csv_bytes, + ) + + st.success("Flexible benchmark dataset is now registered for evaluation.") + except Exception as exc: # noqa: BLE001 + st.error(f"Flexible benchmark alignment failed: {exc}") + + def _render_config_form(self) -> RetrievalConfig: + """Render retrieval configuration form""" + col1, col2, col3 = st.columns(3) + + with col1: + chunk_size = st.number_input("Chunk Size", min_value=100, max_value=2000, value=1000) + chunk_overlap = st.number_input("Chunk Overlap", min_value=0, max_value=500, value=200) + + with col2: + top_k = st.number_input("Top K", min_value=1, max_value=20, value=5) + use_llm_scoring = st.checkbox("Use LLM Scoring", value=False) + + with col3: + embedding_model = st.selectbox("Embedding Model", ["default", "openai", "sentence-transformers"]) + similarity_threshold = st.slider("Similarity Threshold", 0.0, 1.0, 0.0, 0.1) + + llm_model = None + if use_llm_scoring: + llm_model = st.selectbox("LLM Model", ["gpt-4o-mini", "gpt-4o", "gemini-1.5-flash"]) + + return RetrievalConfig( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + top_k=top_k, + use_llm_scoring=use_llm_scoring, + embedding_model=embedding_model, + similarity_threshold=similarity_threshold, + llm_model=llm_model, + ) + + def _run_evaluation(self, dataset_id: str, evaluation_name: str, config: RetrievalConfig): + """Run benchmark evaluation""" + # This would integrate with the existing analyzer + st.info("Evaluation functionality would integrate with the existing DocumentAnalyzer here.") + # For now, create a placeholder evaluation + + # Save evaluation (placeholder) + from ..models.benchmark import EvaluationMetrics + + placeholder_metrics = EvaluationMetrics( + precision_at_k={1: 0.8, 3: 0.7, 5: 0.6}, + recall_at_k={1: 0.2, 3: 0.4, 5: 0.6}, + f1_at_k={1: 0.32, 3: 0.51, 5: 0.6}, + mean_reciprocal_rank=0.75, + mean_average_precision=0.65, + ndcg_at_k={1: 0.8, 3: 0.72, 5: 0.68}, + ) + + evaluation = BenchmarkEvaluation( + dataset_id=dataset_id, + evaluation_name=evaluation_name, + config_hash="", + retrieval_config=config, + evaluation_metrics=placeholder_metrics, + ) + + eval_id = self.benchmark_store.save_evaluation(evaluation) + st.success(f"Evaluation '{evaluation_name}' completed and saved with ID {eval_id}") + + def _render_results_table(self, evaluations): + """Render evaluation results table""" + results_data = [] + for eval in evaluations: + metrics = getattr(eval, "evaluation_metrics", None) + if not metrics: + continue + + eval_name = getattr(eval, "evaluation_name", "Unknown") + dataset_id = getattr(eval, "dataset_id", "N/A") + created_at = getattr(eval, "created_at", None) + + results_data.append( + { + "Model": eval_name, + "Dataset": dataset_id, + "MAP": f"{metrics.mean_average_precision:.3f}", + "MRR": f"{metrics.mean_reciprocal_rank:.3f}", + "P@5": f"{metrics.precision_at_k.get(5, 0):.3f}", + "R@5": f"{metrics.recall_at_k.get(5, 0):.3f}", + "F1@5": f"{metrics.f1_at_k.get(5, 0):.3f}", + "NDCG@5": f"{metrics.ndcg_at_k.get(5, 0):.3f}", + "Date": ( + created_at.strftime("%Y-%m-%d") + if created_at and hasattr(created_at, "strftime") + else (str(created_at) if created_at else "Unknown") + ), + } + ) + + if results_data: + df = pd.DataFrame(results_data) + st.dataframe(df, use_container_width=True) + else: + st.info("No evaluation results to display.") + + def _render_metrics_charts(self, evaluations): + """Render metrics visualization charts""" + if len(evaluations) < 1: + return + + # Prepare data for plotting + chart_data = [] + for eval in evaluations: + metrics = getattr(eval, "evaluation_metrics", None) + if not metrics: + continue + + eval_name = getattr(eval, "evaluation_name", "Unknown") + + # Get all available K values + k_values = sorted(metrics.precision_at_k.keys()) + for k in k_values: + chart_data.append( + { + "Model": eval_name, + "K": k, + "Precision": metrics.precision_at_k.get(k, 0.0), + "Recall": metrics.recall_at_k.get(k, 0.0), + "F1": metrics.f1_at_k.get(k, 0.0), + "NDCG": metrics.ndcg_at_k.get(k, 0.0), + } + ) + + if not chart_data: + return + + df = pd.DataFrame(chart_data) + + # Create charts + col1, col2 = st.columns(2) + + with col1: + fig = px.line( + df, + x="K", + y="Precision", + color="Model", + title="Precision@K by Model", + markers=True, + ) + st.plotly_chart(fig, use_container_width=True, key="results_precision_chart") + + with col2: + fig = px.line( + df, + x="K", + y="Recall", + color="Model", + title="Recall@K by Model", + markers=True, + ) + st.plotly_chart(fig, use_container_width=True, key="results_recall_chart") + + col3, col4 = st.columns(2) + with col3: + fig = px.line( + df, + x="K", + y="F1", + color="Model", + title="F1@K by Model", + markers=True, + ) + st.plotly_chart(fig, use_container_width=True, key="results_f1_chart") + with col4: + fig = px.line( + df, + x="K", + y="NDCG", + color="Model", + title="NDCG@K by Model", + markers=True, + ) + st.plotly_chart(fig, use_container_width=True, key="results_ndcg_chart") + + def _show_dataset_details(self, dataset_id: str): + """Show detailed dataset information""" + dataset = self.benchmark_store.get_dataset(dataset_id) + if dataset: + st.write(f"**Name:** {dataset.name}") + st.write(f"**Description:** {dataset.description}") + st.write(f"**Version:** {dataset.version}") + st.write(f"**Question Set:** {dataset.question_set}") + + # Show ground truth statistics + ground_truth = self.benchmark_store.get_ground_truth(dataset_id) + total_questions = len(ground_truth) + total_chunks = sum(len(chunks) for chunks in ground_truth.values()) + st.write(f"**Questions:** {total_questions}") + st.write(f"**Total Chunks:** {total_chunks}") + + def _delete_dataset(self, dataset_id: str): + """Delete a dataset""" + if st.button(f"Confirm deletion of {dataset_id}", type="secondary"): + if self.benchmark_store.delete_dataset(dataset_id): + st.success(f"Dataset {dataset_id} deleted successfully!") + st.rerun() + else: + st.error("Failed to delete dataset") + + def _render_evaluation_details(self, evaluation): + """Render detailed evaluation information""" + eval_name = getattr(evaluation, "evaluation_name", "Unknown") + st.subheader(f"Evaluation: {eval_name}") + + # Configuration (if available) + if hasattr(evaluation, "retrieval_config") and evaluation.retrieval_config: + config = evaluation.retrieval_config + st.write("**Configuration:**") + if hasattr(config, "model_dump"): + config_dict = config.model_dump() + else: + config_dict = dict(config) if isinstance(config, dict) else {} + st.json(config_dict) + + # Metrics - show only JSON format in details, tables and charts are already shown above + if hasattr(evaluation, "evaluation_metrics") and evaluation.evaluation_metrics: + metrics = evaluation.evaluation_metrics + st.write("**Metrics (JSON):**") + if hasattr(metrics, "model_dump"): + metrics_dict = metrics.model_dump() + else: + metrics_dict = dict(metrics) if isinstance(metrics, dict) else {} + st.json(metrics_dict) + + def _render_annotation_form(self, evaluation: BenchmarkEvaluation): + """Render annotation form for an evaluation""" + st.write(f"Annotating evaluation: **{evaluation.evaluation_name}**") + + # This would show the retrieved chunks for annotation + st.info("Annotation interface would show retrieved chunks here for human evaluation.") + + # Placeholder annotation form + st.text_input("Annotator ID", value="annotator_1") + + if st.button("Save Annotations"): + st.success("Annotations saved! (This is a placeholder)") + + def _render_csv_metrics_table(self, metrics, k_values: Optional[List[int]]): + """Render metrics table for CSV evaluation results""" + st.subheader("Metrics Summary") + + if k_values is None or not k_values: + k_values = sorted(metrics.precision_at_k.keys()) + + # Prepare table data + table_data = { + "Metric": ["MAP", "MRR"], + "Value": [ + f"{metrics.mean_average_precision:.4f}", + f"{metrics.mean_reciprocal_rank:.4f}", + ], + } + + # Add metrics at K + for k in sorted(k_values): + table_data[f"K={k}"] = [ + f"{metrics.precision_at_k.get(k, 0.0):.4f}", + f"{metrics.recall_at_k.get(k, 0.0):.4f}", + ] + + df_summary = pd.DataFrame(table_data) + + # Create detailed metrics table + detailed_data = [] + for k in sorted(k_values): + detailed_data.append( + { + "K": k, + "Precision@K": f"{metrics.precision_at_k.get(k, 0.0):.4f}", + "Recall@K": f"{metrics.recall_at_k.get(k, 0.0):.4f}", + "F1@K": f"{metrics.f1_at_k.get(k, 0.0):.4f}", + "NDCG@K": f"{metrics.ndcg_at_k.get(k, 0.0):.4f}", + } + ) + + df_detailed = pd.DataFrame(detailed_data) + + # Display tables + st.write("**Overall Metrics:**") + st.dataframe(df_summary, use_container_width=True, hide_index=True) + + st.write("**Metrics at K:**") + st.dataframe(df_detailed, use_container_width=True, hide_index=True) + + def _render_csv_metrics_charts(self, metrics, k_values: Optional[List[int]], chart_key_prefix: str = "csv"): + """Render Plotly charts for CSV evaluation results""" + st.subheader("Visualization") + + if k_values is None or not k_values: + k_values = sorted(metrics.precision_at_k.keys()) + + if not k_values: + st.warning("No K values available for plotting.") + return + + # Prepare data for plotting + chart_data = [] + for k in sorted(k_values): + chart_data.append( + { + "K": k, + "Precision": metrics.precision_at_k.get(k, 0.0), + "Recall": metrics.recall_at_k.get(k, 0.0), + "F1": metrics.f1_at_k.get(k, 0.0), + "NDCG": metrics.ndcg_at_k.get(k, 0.0), + } + ) + + df_chart = pd.DataFrame(chart_data) + + # Create charts side by side + col1, col2 = st.columns(2) + + with col1: + fig_precision = px.line( + df_chart, + x="K", + y="Precision", + title="Precision@K", + markers=True, + ) + fig_precision.update_layout( + xaxis_title="K", + yaxis_title="Precision", + yaxis_range=[0, 1.1], + ) + st.plotly_chart( + fig_precision, + use_container_width=True, + key=f"{chart_key_prefix}_precision_{id(metrics)}", + ) + + with col2: + fig_recall = px.line( + df_chart, + x="K", + y="Recall", + title="Recall@K", + markers=True, + ) + fig_recall.update_layout( + xaxis_title="K", + yaxis_title="Recall", + yaxis_range=[0, 1.1], + ) + st.plotly_chart( + fig_recall, + use_container_width=True, + key=f"{chart_key_prefix}_recall_{id(metrics)}", + ) diff --git a/report_analyst_api/main.py b/report_analyst_api/main.py index fef088a6..f1f0f80d 100644 --- a/report_analyst_api/main.py +++ b/report_analyst_api/main.py @@ -176,12 +176,9 @@ async def get_consolidated_results( @app.get("/models", operation_id="get_models") async def get_models(): """List available LLM models for analysis.""" - return [ - {"id": "gpt-4o-mini", "name": "GPT-4o Mini"}, - {"id": "gpt-4o", "name": "GPT-4o"}, - {"id": "gpt-4", "name": "GPT-4"}, - {"id": "gpt-3.5-turbo", "name": "GPT-3.5 Turbo"}, - ] + from report_analyst.core.llm_models import get_models_for_api + + return get_models_for_api() def _get_temp_dir() -> str: diff --git a/report_analyst_enterprise/components/streamlit_component/frontend/public/pdf-viewer.es.js b/report_analyst_enterprise/components/streamlit_component/frontend/public/pdf-viewer.es.js new file mode 100644 index 00000000..e90ea722 --- /dev/null +++ b/report_analyst_enterprise/components/streamlit_component/frontend/public/pdf-viewer.es.js @@ -0,0 +1,985 @@ +class B extends HTMLElement { + constructor() { + super(), this.attachShadow({ mode: "open" }), this._pdfUrl = null, this._pdfData = null, this._chunks = [], this._questions = [], this._selectedQuestionId = null, this._showEvidenceOnly = !1, this._pdfDoc = null, this._currentPage = 1, this._scale = 1.5, this._pdfjsLib = null, this._renderedPages = /* @__PURE__ */ new Map(), this._isLoading = !1, this._highlightedChunkId = null; + } + static get observedAttributes() { + return ["pdf-url", "pdf-data", "chunks", "questions", "selected-question-id", "show-evidence-only"]; + } + connectedCallback() { + this.loadPdfJs().then(() => { + this.render(); + }); + } + disconnectedCallback() { + this._renderedPages.clear(), this._pdfDoc && (this._pdfDoc.destroy(), this._pdfDoc = null); + } + attributeChangedCallback(e, t, s) { + if (t !== s) + try { + e === "pdf-url" ? (this._pdfUrl = s, this._pdfData = null) : e === "pdf-data" ? (this._pdfData = s, this._pdfUrl = null) : e === "chunks" ? this._chunks = s ? JSON.parse(s) : [] : e === "questions" ? this._questions = s ? JSON.parse(s) : [] : e === "selected-question-id" ? this._selectedQuestionId = s : e === "show-evidence-only" && (this._showEvidenceOnly = s === "true" || s === ""), this._skipAttributeRender || this.render(); + } catch (i) { + console.error(`Error parsing ${e}:`, i); + } + } + // Public API: Set PDF URL + setPdfUrl(e) { + this._pdfUrl = e, this._pdfData = null, this.setAttribute("pdf-url", e); + } + // Public API: Set PDF data (base64) + setPdfData(e) { + this._pdfData = e, this._pdfUrl = null, this.setAttribute("pdf-data", e); + } + // Public API: Set chunks + setChunks(e) { + this._chunks = e, this.setAttribute("chunks", JSON.stringify(e)); + } + // Public API: Set questions + setQuestions(e) { + this._questions = e, this.setAttribute("questions", JSON.stringify(e)); + } + // Public API: Set selected question + setSelectedQuestionId(e, t = !1) { + this._selectedQuestionId = e, t ? (this._skipAttributeRender = !0, this.setAttribute("selected-question-id", e || ""), this._skipAttributeRender = !1, this.updateFilterUI()) : this.setAttribute("selected-question-id", e || ""); + } + // Public API: Set evidence filter + setShowEvidenceOnly(e, t = !1) { + this._showEvidenceOnly = e, t ? (this._skipAttributeRender = !0, this.setAttribute("show-evidence-only", e ? "true" : "false"), this._skipAttributeRender = !1, this.updateFilterUI()) : this.setAttribute("show-evidence-only", e ? "true" : "false"); + } + // Update filter UI without full render + updateFilterUI() { + var s, i; + const e = (s = this.shadowRoot) == null ? void 0 : s.getElementById("question-select"); + e && (e.value = this._selectedQuestionId || ""); + const t = (i = this.shadowRoot) == null ? void 0 : i.getElementById("evidence-filter"); + t && (t.checked = this._showEvidenceOnly), this.renderChunkList(); + } + // Render only the chunk list without re-rendering PDF + renderChunkList() { + var s; + const e = (s = this.shadowRoot) == null ? void 0 : s.querySelector(".chunks-list"); + if (!e) return; + const t = this.getFilteredChunks(); + e.innerHTML = t.length === 0 ? '
No chunks to display
' : t.map((i, n) => { + var g, c; + let r = "?"; + i.metadata && (i.metadata.page_number !== void 0 ? r = parseInt(i.metadata.page_number) || "?" : i.metadata.source !== void 0 && (r = parseInt(i.metadata.source) || "?")); + const o = i.is_evidence === !0, l = ((g = i.similarity_score) == null ? void 0 : g.toFixed(3)) || "N/A", h = ((c = i.llm_score) == null ? void 0 : c.toFixed(3)) || "N/A", a = i.text || "", p = a.substring(0, 150) + (a.length > 150 ? "..." : ""); + return ` +
+
+ Chunk ${i.chunk_order !== void 0 ? i.chunk_order + 1 : n + 1} +
+ ${o ? 'Evidence' : ""} + Page ${r} +
+
+
${this.escapeHtml(p)}
+
+ Similarity: ${l} + ${i.llm_score !== null && i.llm_score !== void 0 ? `LLM: ${h}` : ""} +
+
+ `; + }).join(""), this.attachChunkListeners(); + } + // Attach click listeners to chunk items + attachChunkListeners() { + var t; + const e = (t = this.shadowRoot) == null ? void 0 : t.querySelectorAll(".chunk-item"); + e && e.forEach((s) => { + var n; + const i = s.cloneNode(!0); + (n = s.parentNode) == null || n.replaceChild(i, s), i.addEventListener("click", () => { + const r = parseInt(i.dataset.chunkIndex), o = this.getFilteredChunks()[r]; + o && this.navigateToChunk(o); + }); + }); + } + async loadPdfJs() { + if (!this._pdfjsLib) { + if (typeof pdfjsLib > "u") { + const e = document.createElement("script"); + e.src = "https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.min.js", e.async = !0, await new Promise((t, s) => { + e.onload = t, e.onerror = s, document.head.appendChild(e); + }); + } + this._pdfjsLib = window.pdfjsLib || pdfjsLib, this._pdfjsLib.GlobalWorkerOptions && (this._pdfjsLib.GlobalWorkerOptions.workerSrc = "https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js"), this._pdfjsLib.GlobalWorkerOptions && (this._pdfjsLib.GlobalWorkerOptions.cMapUrl = "https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/cmaps/", this._pdfjsLib.GlobalWorkerOptions.cMapPacked = !0); + } + } + async loadPdf() { + if (this._pdfjsLib || await this.loadPdfJs(), this._pdfDoc) + return this._pdfDoc; + this._isLoading = !0, this.updateLoadingDisplay(); + try { + let e; + const t = { + cMapUrl: "https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/cmaps/", + cMapPacked: !0, + standardFontDataUrl: "https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/standard_fonts/" + }; + if (this._pdfData) { + const s = this._pdfData.replace(/^data:application\/pdf;base64,/, ""), i = atob(s), n = new Uint8Array(i.length); + for (let r = 0; r < i.length; r++) + n[r] = i.charCodeAt(r); + e = this._pdfjsLib.getDocument({ + data: n, + ...t + }); + } else if (this._pdfUrl) + e = this._pdfjsLib.getDocument({ + url: this._pdfUrl, + ...t + }); + else + throw new Error("No PDF URL or data provided"); + return this._pdfDoc = await e.promise, this._pdfDoc; + } catch (e) { + throw console.error("Error loading PDF:", e), e; + } + } + updateLoadingDisplay() { + var t; + const e = (t = this.shadowRoot) == null ? void 0 : t.getElementById("viewer-content"); + e && this._isLoading && (e.innerHTML = ` +
+ +
Loading PDF...
+
+ `); + } + getFilteredChunks() { + let e = []; + if (this._selectedQuestionId) { + const t = this._questions.find((s) => s.question_id === this._selectedQuestionId); + t && t.chunks ? e = t.chunks : e = this._chunks.filter((s) => s.question_id === this._selectedQuestionId); + } else + e = this._chunks; + return this._showEvidenceOnly && (e = e.filter((t) => t.is_evidence === !0 || t.is_evidence === 1)), e; + } + async renderPage(e) { + if (this._renderedPages.has(e)) + return this._renderedPages.get(e); + try { + const s = await (await this.loadPdf()).getPage(e), i = s.getViewport({ scale: this._scale }), n = document.createElement("canvas"), r = n.getContext("2d"); + return n.height = i.height, n.width = i.width, await s.render({ + canvasContext: r, + viewport: i + }).promise, this._renderedPages.set(e, n), n; + } catch (t) { + return console.error(`Error rendering page ${e}:`, t), null; + } + } + /** + * Calculate log-likelihood keyness scores for words + * Identifies words that are unusually frequent in this chunk compared to other chunks + * Uses Dunning's log-likelihood (G²) statistic + * @param {string} chunkText - The chunk text to analyze + * @param {Array} allChunks - All chunk texts for comparison + * @returns {Map} Map of word to keyness score + */ + calculateKeyness(e, t = []) { + const s = (c) => c.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((f) => f.length > 2), i = s(e), n = /* @__PURE__ */ new Map(); + if (i.forEach((c) => { + n.set(c, (n.get(c) || 0) + 1); + }), t.length === 0) { + const c = Array.from(n.entries()).sort((f, d) => d[1] - f[1]).slice(0, 10); + return new Map(c); + } + const r = [], o = /* @__PURE__ */ new Map(); + t.forEach((c) => { + s(c.text || c).forEach((d) => { + r.push(d), o.set(d, (o.get(d) || 0) + 1); + }); + }); + const l = /* @__PURE__ */ new Map(), h = i.length, a = r.length, p = h + a; + return (/* @__PURE__ */ new Set([...i, ...r])).forEach((c) => { + const f = n.get(c) || 0, d = o.get(c) || 0; + if (f === 0) + return; + const m = (f + d) * (h / p), v = (f + d) * (a / p); + let u = 0; + f > 0 && m > 0 && (u += 2 * f * Math.log(f / m)), d > 0 && v > 0 && (u += 2 * d * Math.log(d / v)), u > 0.01 && f > m && l.set(c, u); + }), l; + } + /** + * Get word-level importance scores for highlighting + * Uses log-likelihood keyness to identify words unusually frequent in this chunk + * @param {string} chunkText - The chunk text + * @param {Array} allChunks - All chunks for comparison + * @returns {Map} Word to keyness score + */ + getWordImportanceScores(e, t = []) { + return this.calculateKeyness(e, t); + } + /** + * Find text positions for a chunk in the PDF page + * Uses exact matching first, falls back to embedding-based semantic matching + * @param {Object} page - PDF.js page object + * @param {string} chunkText - The chunk text to find + * @param {Object} viewport - PDF.js viewport object + * @param {Array} allChunks - All chunks for context (optional, for TF-IDF) + * @returns {Array} Array of bounding boxes {x, y, width, height, wordScores} in viewport coordinates + */ + async findChunkTextPositions(e, t, s, i = []) { + const n = await this.findChunkTextPositionsExact(e, t, s); + if (n.length > 0) { + const r = this.getWordImportanceScores(t, i); + return n.forEach((o) => { + o.wordScores = r; + }), n; + } + return []; + } + /** + * Exact text matching (original implementation) + * @param {Object} page - PDF.js page object + * @param {string} chunkText - The chunk text to find + * @param {Object} viewport - PDF.js viewport object + * @returns {Array} Array of bounding boxes + */ + async findChunkTextPositionsExact(e, t, s) { + try { + const i = await e.getTextContent(); + if (!i || !i.items || i.items.length === 0) + return console.warn("No text content found on page"), []; + const n = (g) => g.toLowerCase().trim().replace(/\s+/g, " "), r = n(t); + if (!r || r.length < 10) + return console.warn("Chunk text too short for reliable matching"), []; + const o = i.items, l = o.map((g) => g.str).join(" "), h = n(l); + let a = h.indexOf(r), p = r; + if (a === -1) { + const g = r.split(" "), c = g.slice(0, Math.min(20, g.length)).join(" "); + a = h.indexOf(c), a !== -1 && (p = c); + } + if (a === -1) { + const g = r.split(" "), c = g.slice(0, Math.min(10, g.length)).join(" "); + a = h.indexOf(c), a !== -1 && (p = c); + } + return a === -1 ? (console.warn(`Chunk text not found on page: "${t.substring(0, 50)}..."`), []) : this.findTextItemPositions(o, p, a, h, s, n); + } catch (i) { + return console.error("Error finding chunk text positions:", i), []; + } + } + /** + * Find text item positions that match the search text + * @param {Array} textItems - Array of text items from PDF.js + * @param {string} searchText - Normalized text to search for + * @param {number} textIndex - Character index where searchText was found in normalized text + * @param {string} normalizedAllText - Full normalized text from all items + * @param {Object} viewport - PDF.js viewport object + * @param {Function} normalizeText - Text normalization function + * @returns {Array} Array of bounding boxes + */ + findTextItemPositions(e, t, s, i, n, r) { + const o = []; + let l = 0; + const h = []; + for (let a = 0; a < e.length; a++) { + const p = e[a], g = r(p.str), c = g.length + 1; + if (l + g.length >= s && l <= s + t.length && h.push(p), l += c, l > s + t.length) + break; + } + if (h.length === 0) { + const a = t.split(" ").slice(0, 5).join(" "); + let p = ""; + for (const g of e) { + const c = r(g.str); + if (p += c + " ", h.push(g), r(p).includes(a)) + break; + if (h.length > 50) { + h.length = 0; + break; + } + } + } + if (h.length > 0) { + const a = this.calculateBoundingBox(h, n); + a && a.width > 0 && a.height > 0 && o.push(a); + } + return o; + } + /** + * Calculate bounding box from text items and convert to viewport coordinates + * @param {Array} textItems - Array of text items that form the match + * @param {Object} viewport - PDF.js viewport object + * @returns {Object|null} Bounding box {x, y, width, height} in viewport coordinates, or null + */ + calculateBoundingBox(e, t) { + if (!e || e.length === 0) + return null; + let s = 1 / 0, i = 1 / 0, n = -1 / 0, r = -1 / 0; + for (const d of e) + if (d.transform && d.transform.length >= 6) { + const m = d.transform[4], v = d.transform[5], u = d.width || 0, y = d.height || Math.abs(d.transform[3]) || 12; + s = Math.min(s, m), i = Math.min(i, v), n = Math.max(n, m + u), r = Math.max(r, v + y); + } else if (d.x !== void 0 && d.y !== void 0) { + const m = d.x, v = d.y, u = d.width || 0, y = d.height || 12; + s = Math.min(s, m), i = Math.min(i, v), n = Math.max(n, m + u), r = Math.max(r, v + y); + } + if (s === 1 / 0 || i === 1 / 0) + return null; + let o, l, h, a; + if (t.convertToViewportPoint) + [o, l] = t.convertToViewportPoint(s, i), [h, a] = t.convertToViewportPoint(n, r); + else { + const d = t.height / t.scale; + o = s * t.scale, l = (d - r) * t.scale, h = n * t.scale, a = (d - i) * t.scale; + } + const p = Math.min(o, h), g = Math.min(l, a), c = Math.abs(h - o), f = Math.abs(a - l); + return c < 1 || f < 1 ? null : { x: p, y: g, width: c, height: f }; + } + /** + * Add word-level highlights based on TF-IDF scores + * Highlights individual words within the matched text region + * @param {HTMLElement} container - Container to add highlights to + * @param {Object} page - PDF.js page object + * @param {Object} bbox - Bounding box of matched text + * @param {Map} wordScores - Map of word to TF-IDF score + * @param {Object} viewport - PDF.js viewport + * @param {boolean} isEvidence - Whether this is an evidence chunk + */ + async addWordLevelHighlights(e, t, s, i, n, r) { + try { + const o = await t.getTextContent(); + if (!o || !o.items) + return; + const l = /* @__PURE__ */ new Set([ + "the", + "be", + "to", + "of", + "and", + "a", + "in", + "that", + "have", + "i", + "it", + "for", + "not", + "on", + "with", + "he", + "as", + "you", + "do", + "at", + "this", + "but", + "his", + "by", + "from", + "they", + "we", + "say", + "her", + "she", + "or", + "an", + "will", + "my", + "one", + "all", + "would", + "there", + "their", + "what", + "so", + "up", + "out", + "if", + "about", + "who", + "get", + "which", + "go", + "me", + "when", + "make", + "can", + "like", + "time", + "no", + "just", + "him", + "know", + "take", + "people", + "into", + "year", + "your", + "good", + "some", + "could", + "them", + "see", + "other", + "than", + "then", + "now", + "look", + "only", + "come", + "its", + "over", + "think", + "also", + "back", + "after", + "use", + "two", + "how", + "our", + "work", + "first", + "well", + "way", + "even", + "new", + "want", + "because", + "any", + "these", + "give", + "day", + "most", + "us", + "is", + "are", + "was", + "were", + "been", + "being", + "has", + "had", + "does", + "did", + "may", + "might", + "must", + "shall", + "should", + "could", + "would", + "can", + "cannot", + "will", + "shall" + ]); + let h = i; + i instanceof Map || (h = new Map(Object.entries(i || {}))); + const a = Array.from(h.entries()).sort((w, x) => x[1] - w[1]).slice(0, 10); + if (a.length === 0) { + console.warn("No key words found for highlighting - keyness scores may be empty. WordScores:", h); + return; + } + console.log(`Found ${a.length} key words for highlighting:`, a.map(([w, x]) => `${w}(${x.toFixed(3)})`)); + const p = a[0][1], g = a[a.length - 1][1], c = p - g || 1, f = (w) => w.toLowerCase().replace(/[^\w]/g, ""), d = /* @__PURE__ */ new Set(), m = /* @__PURE__ */ new Map(); + if (a.forEach(([w, x]) => { + const k = f(w); + k.length >= 3 && (d.add(k), m.set(k, x)); + }), d.size === 0) + return; + const v = 0.1, u = s.x - s.width * v, y = s.x + s.width + s.width * v, T = s.y - s.height * v, j = s.y + s.height + s.height * v, D = n.height / n.scale, O = (w, x, k, C) => { + if (n.convertToViewportPoint) { + const [L, q] = n.convertToViewportPoint(w, x), W = w + k, E = x + C, [P, I] = n.convertToViewportPoint(W, E); + return { + x: L, + y: q, + width: Math.abs(P - L), + height: Math.abs(I - q) + }; + } else + return { + x: w * n.scale, + y: (D - (x + C)) * n.scale, + width: k * n.scale, + height: C * n.scale + }; + }; + let S = 0; + const N = 50; + for (const w of o.items) { + if (S >= N) break; + if (!w.transform || w.transform.length < 6) continue; + const x = w.transform[4], k = w.transform[5], C = w.width || 0, L = w.height || Math.abs(w.transform[3]) || 12, q = x + C, W = k + L, E = O(x, k, C, L), P = E.x, I = E.y, F = E.width, R = E.height; + if (P < u || P + F > y || I < T || I + R > j) + continue; + const M = f(w.str); + if (d.has(M)) { + const $ = m.get(M), z = 0.5 + ($ - g) / c * 0.4, _ = document.createElement("div"); + _.className = `word-highlight ${r ? "evidence-word" : ""}`, _.style.left = `${P / n.width * 100}%`, _.style.top = `${I / n.height * 100}%`, _.style.width = `${F / n.width * 100}%`, _.style.height = `${R / n.height * 100}%`, _.style.opacity = z, _.style.backgroundColor = r ? "rgba(255, 200, 0, 0.7)" : "rgba(255, 255, 0, 0.6)", _.style.borderRadius = "2px", _.title = `Important word: "${w.str}" (Keyness: ${$.toFixed(3)})`, e.appendChild(_), S++; + } else + for (const $ of d) + if (M.startsWith($) || M.endsWith($)) { + const A = m.get($), _ = 0.5 + (A - g) / c * 0.4, b = document.createElement("div"); + b.className = `word-highlight ${r ? "evidence-word" : ""}`, b.style.left = `${P / n.width * 100}%`, b.style.top = `${I / n.height * 100}%`, b.style.width = `${F / n.width * 100}%`, b.style.height = `${R / n.height * 100}%`, b.style.opacity = _, b.style.backgroundColor = r ? "rgba(255, 200, 0, 0.7)" : "rgba(255, 255, 0, 0.6)", b.style.borderRadius = "2px", b.title = `Important word: "${w.str}" (Keyness: ${A.toFixed(3)})`, e.appendChild(b), S++; + break; + } + } + console.log(`Added ${S} word highlights for ${a.length} key words`); + } catch (o) { + console.error("Error adding word-level highlights:", o); + } + } + async navigateToPage(e) { + const t = this._selectedQuestionId, s = this._showEvidenceOnly; + this._currentPage = e, await this.render(), this._selectedQuestionId = t, this._showEvidenceOnly = s, requestAnimationFrame(() => { + var r, o; + const i = (r = this.shadowRoot) == null ? void 0 : r.getElementById("question-select"); + i && (i.value = t || ""); + const n = (o = this.shadowRoot) == null ? void 0 : o.getElementById("evidence-filter"); + n && (n.checked = s); + }); + } + async navigateToChunk(e) { + const t = this._selectedQuestionId, s = this._showEvidenceOnly; + let i = 1; + if (e.metadata && (e.metadata.page_number !== void 0 ? i = parseInt(e.metadata.page_number) || 1 : e.metadata.source !== void 0 && (i = parseInt(e.metadata.source) || 1)), this._pdfDoc) { + const n = this._pdfDoc.numPages; + i < 1 && (i = 1), i > n && (i = n); + } + await this.navigateToPage(i), this._selectedQuestionId = t, this._showEvidenceOnly = s, this.dispatchEvent(new CustomEvent("chunk-selected", { + detail: { chunk: e, pageNum: i }, + bubbles: !0, + composed: !0 + })); + } + // Public API: Navigate to chunk by ID (for Streamlit communication) + // chunkId format: "question_id_chunk_order" (e.g., "tcfd_1_0") + // Note: question_id may contain underscores, so we split from the right + async navigateToChunkById(e) { + if (!e) return; + const t = e.lastIndexOf("_"); + if (t === -1) { + console.warn(`Invalid chunk ID format: ${e}. Expected format: "question_id_chunk_order"`); + return; + } + const s = e.substring(0, t), i = e.substring(t + 1), n = parseInt(i); + if (isNaN(n)) { + console.warn(`Invalid chunk order in chunk ID: ${e} (parsed as: ${i})`); + return; + } + const r = this._chunks.find((l) => { + const h = l.question_id || "", a = l.chunk_order !== void 0 ? l.chunk_order : -1; + return h === s && (a === n || a === n - 1 || a === n + 1); + }); + if (!r) { + console.warn(`Chunk not found for ID: ${e} (question_id: ${s}, chunk_order: ${n})`), console.debug("Available chunks:", this._chunks.map((l) => ({ + question_id: l.question_id, + chunk_order: l.chunk_order + }))); + return; + } + const o = this._showEvidenceOnly; + this.setSelectedQuestionId(s), await new Promise((l) => setTimeout(l, 100)), await this.navigateToChunk(r), this._showEvidenceOnly = o, this._highlightedChunkId = e; + } + async render() { + if (!this.shadowRoot) return; + const e = this._selectedQuestionId, t = this._showEvidenceOnly, s = this.getFilteredChunks(), i = {}; + s.forEach((o) => { + let l = 1; + o.metadata && (o.metadata.page_number !== void 0 ? l = parseInt(o.metadata.page_number) || 1 : o.metadata.source !== void 0 && (l = parseInt(o.metadata.source) || 1)), i[l] || (i[l] = []), i[l].push(o); + }); + const n = ` + + `, r = ` +
+
+
+

Chunks by Question

+
+ +
+
+ + +
+
+
+ ${s.length === 0 ? '
No chunks to display
' : ""} + ${s.map((o, l) => { + var d, m; + let h = "?"; + o.metadata && (o.metadata.page_number !== void 0 ? h = parseInt(o.metadata.page_number) || "?" : o.metadata.source !== void 0 && (h = parseInt(o.metadata.source) || "?")); + const a = o.is_evidence === !0, p = ((d = o.similarity_score) == null ? void 0 : d.toFixed(3)) || "N/A", g = ((m = o.llm_score) == null ? void 0 : m.toFixed(3)) || "N/A", c = o.text || "", f = c.substring(0, 150) + (c.length > 150 ? "..." : ""); + return ` +
+
+ Chunk ${o.chunk_order !== void 0 ? o.chunk_order + 1 : l + 1} +
+ ${a ? 'Evidence' : ""} + Page ${h} +
+
+
${this.escapeHtml(f)}
+
+ Similarity: ${p} + ${o.llm_score !== null && o.llm_score !== void 0 ? `LLM: ${g}` : ""} +
+
+ `; + }).join("")} +
+
+
+
+ + + Page ${this._currentPage} of - + + +
+
+
Loading PDF...
+
+
+
+ `; + this.shadowRoot.innerHTML = n + r, this._selectedQuestionId = e, this._showEvidenceOnly = t, this.setupEventListeners(), setTimeout(() => { + const o = this.shadowRoot.getElementById("question-select"); + o && this._selectedQuestionId !== void 0 && (o.value = this._selectedQuestionId || ""); + const l = this.shadowRoot.getElementById("evidence-filter"); + l && this._showEvidenceOnly !== void 0 && (l.checked = this._showEvidenceOnly); + }, 0), this.loadAndRenderPdf(); + } + escapeHtml(e) { + const t = document.createElement("div"); + return t.textContent = e, t.innerHTML; + } + setupEventListeners() { + const e = this.shadowRoot.getElementById("question-select"); + e && e.addEventListener("change", (n) => { + this.setSelectedQuestionId(n.target.value || null, !0); + }); + const t = this.shadowRoot.getElementById("evidence-filter"); + t && t.addEventListener("change", (n) => { + this.setShowEvidenceOnly(n.target.checked, !0); + }), this.attachChunkListeners(); + const s = this.shadowRoot.getElementById("prev-page"), i = this.shadowRoot.getElementById("next-page"); + s && s.addEventListener("click", () => { + this._currentPage > 1 && this.navigateToPage(this._currentPage - 1); + }), i && i.addEventListener("click", async () => { + if (this._pdfDoc) { + const n = this._pdfDoc.numPages; + this._currentPage < n && await this.navigateToPage(this._currentPage + 1); + } + }); + } + async loadAndRenderPdf() { + try { + this._isLoading = !0, this.updateLoadingDisplay(); + const t = (await this.loadPdf()).numPages, s = this.shadowRoot.getElementById("total-pages"); + s && (s.textContent = t), await this.renderCurrentPage(), this._isLoading = !1; + } catch (e) { + this._isLoading = !1; + const t = this.shadowRoot.getElementById("viewer-content"); + t && (t.innerHTML = `
Error loading PDF: ${e.message}
`); + } + } + async renderCurrentPage() { + const e = this.shadowRoot.getElementById("viewer-content"); + if (e) + try { + const t = await this.loadPdf(), s = t.numPages; + this._currentPage < 1 && (this._currentPage = 1), this._currentPage > s && (this._currentPage = s); + const i = this.shadowRoot.getElementById("current-page"); + i && (i.textContent = this._currentPage); + const n = await this.renderPage(this._currentPage); + if (!n) { + e.innerHTML = '
Error rendering page
'; + return; + } + const r = await t.getPage(this._currentPage), o = r.getViewport({ scale: this._scale }), l = this.getFilteredChunks(), h = l.filter((c) => { + let f = 1; + return c.metadata && (c.metadata.page_number !== void 0 ? f = parseInt(c.metadata.page_number) || 1 : c.metadata.source !== void 0 && (f = parseInt(c.metadata.source) || 1)), f === this._currentPage; + }), a = document.createElement("div"); + a.className = "page-container"; + const p = document.createElement("canvas"); + if (p.className = "page-canvas", p.width = n.width, p.height = n.height, p.getContext("2d").drawImage(n, 0, 0), a.appendChild(p), h.length > 0) { + const c = document.createElement("div"); + c.className = "page-highlights"; + const f = l.map((d) => ({ text: d.text || "" })); + for (const d of h) { + const m = d.text || ""; + if (!m || m.trim().length === 0) + continue; + const v = await this.findChunkTextPositions( + r, + m, + o, + f + ); + if (v.length > 0) + v.forEach((u) => { + const y = document.createElement("div"); + y.className = `highlight ${d.is_evidence === !0 || d.is_evidence === 1 ? "evidence" : ""}`; + const T = u.x / o.width * 100, j = u.y / o.height * 100, D = u.width / o.width * 100, O = u.height / o.height * 100; + y.style.left = `${T}%`, y.style.top = `${j}%`, y.style.width = `${D}%`, y.style.height = `${O}%`, y.title = d.is_evidence === !0 || d.is_evidence === 1 ? `Evidence chunk: ${m.substring(0, 50)}...` : `Chunk: ${m.substring(0, 50)}...`, c.appendChild(y), u.wordScores && (u.wordScores instanceof Map ? u.wordScores.size > 0 : Object.keys(u.wordScores || {}).length > 0) ? (console.log(`Adding word highlights for chunk with ${u.wordScores instanceof Map ? u.wordScores.size : Object.keys(u.wordScores || {}).length} word scores`), this.addWordLevelHighlights( + c, + r, + u, + u.wordScores, + o, + d.is_evidence === !0 || d.is_evidence === 1 + )) : console.warn("No wordScores found for chunk, skipping word highlights"); + }); + else { + console.warn(`Could not find text position for chunk on page ${this._currentPage}`); + const u = document.createElement("div"); + u.className = `highlight ${d.is_evidence === !0 || d.is_evidence === 1 ? "evidence" : ""}`, u.style.top = "5%", u.style.left = "5%", u.style.width = "10px", u.style.height = "10px", u.style.borderRadius = "50%", u.title = "Chunk text position not found", c.appendChild(u); + } + } + a.appendChild(c); + } + e.innerHTML = "", e.appendChild(a); + } catch (t) { + console.error("Error rendering current page:", t), e.innerHTML = `
Error rendering page: ${t.message}
`; + } + } +} +customElements.get("pdf-viewer-with-chunks") || customElements.define("pdf-viewer-with-chunks", B); +export { + B as default +}; diff --git a/requirements.txt b/requirements.txt index 3cbec051..31a8460d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ aiofiles==23.2.1 aioresponses==0.7.7 -aiohappyeyeballs==2.4.4 -aiohttp==3.11.11 +aiohappyeyeballs==2.5.0 +aiohttp==3.14.1 aioitertools==0.12.0 -aiosignal==1.3.2 +aiosignal==1.4.0 aiosqlite==0.21.0 alembic==1.15.1 altair==5.5.0 @@ -11,7 +11,7 @@ annotated-types==0.7.0 anyio==4.8.0 asgiref==3.8.1 attrs==25.1.0 -Authlib==1.5.1 +Authlib==1.6.12 backoff==2.2.1 bcrypt==4.2.1 beautifulsoup4==4.13.3 @@ -21,14 +21,14 @@ blinker==1.9.0 build==1.2.2.post1 cachetools==5.5.1 certifi==2024.12.14 -cffi==1.17.1 +cffi==2.1.0 charset-normalizer==3.4.1 chroma-hnswlib==0.7.3 chromadb==0.4.24 click==8.1.8 coloredlogs==15.0.1 contourpy==1.3.1 -cryptography==44.0.2 +cryptography==48.0.1 cycler==0.12.1 dataclasses-json==0.6.7 Deprecated==1.2.18 @@ -37,13 +37,13 @@ distro==1.9.0 duckduckgo_search==7.5.1 durationpy==0.9 faiss-cpu==1.12.0 -fastapi==0.115.11 +fastapi==0.139.2 ffmpy==0.5.0 -filelock==3.17.0 +filelock==3.20.3 filetype==1.2.0 flake8==7.1.2 flatbuffers==25.1.24 -fonttools==4.55.8 +fonttools==4.60.2 frozenlist==1.5.0 fsspec==2025.2.0 gitdb==4.0.12 @@ -51,8 +51,6 @@ GitPython==3.1.50 google-auth==2.38.0 google-generativeai>=0.3.0 googleapis-common-protos==1.66.0 -gradio==5.20.1 -gradio_client==1.7.2 graphql-core==3.2.6 greenlet==3.1.1 groovy==0.1.2 @@ -65,7 +63,7 @@ httpx==0.27.0 httpx-sse==0.4.0 huggingface-hub==0.28.1 humanfriendly==10.0 -idna==3.10 +idna==3.15 importlib_metadata==7.2.1 importlib_resources==6.5.2 iniconfig==2.0.0 @@ -81,32 +79,23 @@ kiwisolver==1.4.8 kubernetes==32.0.0 langchain==0.3.20 langchain-community==0.3.19 -langchain-core==0.3.43 +langchain-core==0.3.85 langchain-openai==0.3.8 langchain-text-splitters==0.3.6 -langsmith==0.3.13 +langsmith==0.3.45 llama-cloud==0.1.11 llama-cloud-services==0.6.0 -llama-index==0.12.16 -llama-index-agent-openai==0.4.3 -llama-index-cli==0.4.0 -llama-index-core==0.12.16.post1 -llama-index-embeddings-openai==0.3.1 -llama-index-indices-managed-llama-cloud==0.6.4 -llama-index-llms-gemini==0.4.12 -llama-index-llms-openai==0.3.18 -llama-index-multi-modal-llms-openai==0.4.3 -llama-index-program-openai==0.3.1 -llama-index-question-gen-openai==0.3.0 -llama-index-readers-file==0.4.4 -llama-index-readers-llama-parse==0.4.0 -llama-parse==0.6.0 -lxml==5.3.1 -Mako==1.3.9 -Markdown==3.5.2 +llama-index-core==0.13.6 +llama-index-embeddings-openai==0.5.2 +llama-index-llms-gemini==0.6.0 +llama-index-llms-openai==0.5.6 +llama-index-readers-file==0.5.6 +lxml==6.1.0 +Mako==1.3.12 +Markdown==3.8.1 markdown-it-py==3.0.0 MarkupSafe==2.1.5 -marshmallow==3.26.1 +marshmallow==3.26.2 matplotlib==3.10.0 mccabe==0.7.0 mdurl==0.1.2 @@ -119,15 +108,15 @@ nats-py==2.10.0 narwhals==1.25.0 nest-asyncio==1.6.0 networkx==3.4.2 -nltk==3.9.1 +nltk==3.10.0 numpy==1.26.4 oauthlib==3.2.2 onnxruntime==1.20.1 -openai==1.61.1 +openai==1.81.0 openinference-instrumentation==0.1.23 openinference-instrumentation-langchain==0.1.35 openinference-semantic-conventions==0.1.14 -orjson==3.10.15 +orjson==3.11.6 overrides==7.7.0 packaging==23.2 pandas==2.2.3 @@ -143,26 +132,26 @@ protobuf==5.29.6 psutil==7.0.0 pulsar-client==3.6.1 pyarrow==19.0.0 -pyasn1==0.6.1 +pyasn1==0.6.3 pyasn1_modules==0.4.1 pycodestyle==2.12.1 pycparser==2.22 -pydantic==2.10.4 +pydantic==2.11.5 pydantic-settings==2.8.1 -pydantic_core==2.27.2 +pydantic_core==2.33.2 pydeck==0.9.1 pydub==0.25.1 pyflakes==3.2.0 -Pygments==2.19.1 +Pygments==2.20.0 PyMuPDF==1.24.0 PyMuPDFb==1.24.0 pyparsing==3.2.1 -pypdf==5.2.0 +pypdf==6.13.3 PyPika==0.48.9 pyproject_hooks==1.2.0 psycopg2-binary==2.9.10 -pytest==8.3.5 -pytest-asyncio==0.25.3 +pytest==9.0.3 +pytest-asyncio==1.4.0 pytest-cov==7.0.0 diff-cover>=9.0.0 pytest-env==1.1.1 @@ -172,8 +161,8 @@ pytest-randomly==3.16.0 pytest-socket==0.7.0 pytest-timeout==2.3.1 python-dateutil==2.9.0.post0 -python-dotenv==1.0.1 -python-multipart==0.0.20 +python-dotenv==1.2.2 +python-multipart==0.0.31 pytz==2025.1 PyYAML==6.0.1 referencing==0.36.2 @@ -190,17 +179,17 @@ safehttpx==0.1.6 scikit-learn==1.6.1 scipy==1.15.2 semantic-version==2.10.0 -setuptools==78.1.1 +setuptools==80.9.0 shellingham==1.5.4 six==1.17.0 smmap==5.0.2 sniffio==1.3.1 -soupsieve==2.6 +soupsieve==2.8.4 SQLAlchemy==2.0.37 sqlean.py==3.47.0 sqlite-vec==0.1.6 sqlite-vss==0.1.2 -starlette==0.46.1 +starlette==1.3.1 strawberry-graphql==0.262.1 streamlit==1.52.0 streamlit-card==1.0.2 diff --git a/sample_datasets/tcfd_sample_benchmark.yaml b/sample_datasets/tcfd_sample_benchmark.yaml new file mode 100644 index 00000000..ac21503c --- /dev/null +++ b/sample_datasets/tcfd_sample_benchmark.yaml @@ -0,0 +1,77 @@ +dataset_id: "tcfd_sample_v1" +name: "TCFD Sample Benchmark Dataset" +description: "Sample dataset for testing TCFD question benchmarking" +version: "1.0" +question_set: "tcfd" +created_at: "2024-01-15" +questions: + - question_id: "tcfd_1" + question_text: "What are the climate-related risks that the organization has identified over the short, medium, and long term?" + ground_truth_chunks: + - chunk_id: "chunk_001" + relevance_score: 1.0 + is_evidence: true + evidence_order: 1 + annotation_notes: "Contains specific climate risk identification and timeframes" + - chunk_id: "chunk_015" + relevance_score: 0.9 + is_evidence: true + evidence_order: 2 + annotation_notes: "Details risk assessment methodology" + - chunk_id: "chunk_042" + relevance_score: 0.3 + is_evidence: false + annotation_notes: "Mentions climate risks but lacks specificity" + - chunk_id: "chunk_067" + relevance_score: 0.0 + is_evidence: false + annotation_notes: "Unrelated content about financial performance" + + - question_id: "tcfd_2" + question_text: "How do climate-related risks and opportunities influence the organization's strategy and financial planning?" + ground_truth_chunks: + - chunk_id: "chunk_023" + relevance_score: 1.0 + is_evidence: true + evidence_order: 1 + annotation_notes: "Direct discussion of climate impact on strategy" + - chunk_id: "chunk_034" + relevance_score: 0.8 + is_evidence: true + evidence_order: 2 + annotation_notes: "Financial planning considerations for climate risks" + - chunk_id: "chunk_089" + relevance_score: 0.6 + is_evidence: false + annotation_notes: "General strategic planning discussion" + - chunk_id: "chunk_105" + relevance_score: 0.0 + is_evidence: false + annotation_notes: "Operational details unrelated to climate strategy" + + - question_id: "tcfd_3" + question_text: "What are the organization's processes for identifying, assessing, and managing climate-related risks?" + ground_truth_chunks: + - chunk_id: "chunk_012" + relevance_score: 1.0 + is_evidence: true + evidence_order: 1 + annotation_notes: "Detailed risk management process description" + - chunk_id: "chunk_028" + relevance_score: 0.9 + is_evidence: true + evidence_order: 2 + annotation_notes: "Risk assessment methodology and tools" + - chunk_id: "chunk_055" + relevance_score: 0.7 + is_evidence: true + evidence_order: 3 + annotation_notes: "Governance structure for risk management" + - chunk_id: "chunk_078" + relevance_score: 0.4 + is_evidence: false + annotation_notes: "General risk management discussion" + - chunk_id: "chunk_091" + relevance_score: 0.0 + is_evidence: false + annotation_notes: "IT risk management - not climate related" diff --git a/scripts/README_CLIMRETRIEVE.md b/scripts/README_CLIMRETRIEVE.md new file mode 100644 index 00000000..5c52539b --- /dev/null +++ b/scripts/README_CLIMRETRIEVE.md @@ -0,0 +1,130 @@ +# ClimRetrieve Benchmark Testing + +This script downloads and tests ClimRetrieve benchmark datasets from GitHub. + +## Datasets + +- **Reference Dataset**: Expert-Annotated Relevant Sources Dataset + - Location: `Expert-Annotated Relevant Sources Dataset/` in the ClimRetrieve repo + - This is the ground truth dataset + +- **Input Dataset**: Report-Level Dataset + - Location: `Report-Level Dataset/` in the ClimRetrieve repo + - This is the dataset to evaluate against the reference + +## Usage + +### Basic Usage (Downloads and Tests) + +```bash +python scripts/test_climretrieve_benchmark.py +``` + +This will: +1. Download datasets from GitHub to `data/climretrieve/` +2. Load both datasets +3. Run evaluation comparison +4. Display metrics + +### Using Existing Files + +If you already have the datasets downloaded: + +```bash +python scripts/test_climretrieve_benchmark.py --skip-download +``` + +### Providing Custom Paths + +```bash +python scripts/test_climretrieve_benchmark.py \ + --reference-path path/to/reference.csv \ + --input-path path/to/input.csv +``` + +### Custom K Values + +```bash +python scripts/test_climretrieve_benchmark.py --k-values 1 5 10 20 +``` + +## Generic CSV Evaluation Script + +In addition to the ClimRetrieve-specific script, there is a generic helper for +evaluating **any** pair of benchmark datasets stored as CSV files: + +- Script: `scripts/evaluate_benchmark_from_csv.py` +- Loader: `load_flexible_dataset_from_csv` (auto-detects IR vs IE datasets and + handles flexible column names) +- Engine: `EvaluationEngine` (computes precision@K, recall@K, F1@K, NDCG@K, + MAP, MRR) + +### Expected CSV Formats + +The flexible loader supports multiple column name variants and will +auto-detect the dataset type: + +- **IR-style (information retrieval) datasets** + Typical columns: + + - `query_id` or `question_id` or `qid` + - `chunk_id` or `chunk` or `cid` + - `position` or `rank` or `order` (1-indexed) + - `score` or `relevance_score` or `similarity_score` + +- **IE-style (information extraction / QA) datasets** + Typical columns: + + - `question_id` or `query_id` + - `answer` or `analysis` or `text` or `response` + - Optional: `category` / `class` / `label` / `type` + +Anything beyond these required fields is preserved as metadata but does not +affect the metrics. + +### Usage with Local CSVs + +Run the generic evaluator via `python -m`: + +```bash +python -m scripts.evaluate_benchmark_from_csv \ + --reference path/to/reference.csv \ + --input path/to/results.csv \ + --k-values 1 3 5 10 \ + --output metrics.json +``` + +This will: + +1. Load both CSVs with `load_flexible_dataset_from_csv`. +2. Use `EvaluationEngine.compare_flexible_datasets` to compute: + - Precision@K, Recall@K, F1@K, NDCG@K + - Mean Average Precision (MAP) + - Mean Reciprocal Rank (MRR) +3. Print a human-readable table to stdout. +4. Optionally write the metrics as JSON to `--output` (one-level dict with + per-K metrics and overall MAP/MRR). + +## Output + +The script will display: +- Dataset inspection (columns, sample rows) +- Loading progress +- Evaluation metrics: + - Mean Average Precision (MAP) + - Mean Reciprocal Rank (MRR) + - Precision@K, Recall@K, F1@K, NDCG@K + +## Data Directory + +Datasets are stored in `data/climretrieve/` (ignored by git). + +The script automatically: +- Creates the directory if it doesn't exist +- Downloads files if they don't exist +- Reuses existing files if present + +## Requirements + +- `requests` library (already in requirements.txt) +- Internet connection for downloading from GitHub diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..776158f4 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,11 @@ +""" +Utility and benchmarking scripts for Open Sustainability Analyst. + +This package allows running helper scripts via: + + python -m scripts. ... + +For example: + + python -m scripts.evaluate_benchmark_from_csv --help +""" diff --git a/scripts/align_benchmark_datasets.py b/scripts/align_benchmark_datasets.py new file mode 100755 index 00000000..29580649 --- /dev/null +++ b/scripts/align_benchmark_datasets.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +""" +Script to align ground truth and benchmark datasets for consistent evaluation. + +This CLI is now a thin wrapper over the DatasetMapper abstraction in +report_analyst.core.benchmark.dataset_mapper so that the same logic can be +used from both the command line and the Streamlit app. +""" + +import argparse +import logging +from pathlib import Path + +import pandas as pd + +from report_analyst.core.benchmark.dataset_mapper import DatasetMapperFactory + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Align ground truth and benchmark datasets for consistent evaluation") + parser.add_argument( + "--dataset-id", + type=str, + default="climretrieve", + help="Dataset identifier used to select the mapping configuration (default: climretrieve)", + ) + parser.add_argument( + "--ground-truth", + type=str, + required=True, + help="Path to ground truth dataset (CSV or Excel)", + ) + parser.add_argument( + "--benchmark-results", + type=str, + required=True, + help="Path to benchmark results dataset (CSV)", + ) + parser.add_argument( + "--output-ground-truth", + type=str, + help="Output path for transformed ground truth CSV (default: _aligned.csv)", + ) + parser.add_argument( + "--output-benchmark", + type=str, + help="Output path for transformed benchmark CSV (default: _aligned.csv)", + ) + + args = parser.parse_args() + + dataset_id: str = args.dataset_id + mapper = DatasetMapperFactory.get_mapper(dataset_id) + logger.info("Using dataset mapper for '%s'", dataset_id) + + # Load ground truth + gt_path = Path(args.ground_truth) + logger.info("Loading ground truth from: %s", gt_path) + if gt_path.suffix in [".xlsx", ".xls"]: + df_gt = pd.read_excel(gt_path) + else: + df_gt = pd.read_csv(gt_path) + + # Align ground truth + output_gt_path = args.output_ground_truth or str(gt_path).replace(".csv", "_aligned.csv").replace( + ".xlsx", "_aligned.csv" + ).replace(".xls", "_aligned.csv") + df_gt_aligned = mapper.align_ground_truth(df_gt) + df_gt_aligned.to_csv(output_gt_path, index=False) + logger.info("Saved transformed ground truth to: %s", output_gt_path) + + # Load benchmark results + benchmark_path = Path(args.benchmark_results) + logger.info("Loading benchmark results from: %s", benchmark_path) + df_benchmark = pd.read_csv(benchmark_path) + + # Align benchmark results + output_benchmark_path = args.output_benchmark or str(benchmark_path).replace(".csv", "_aligned.csv") + df_benchmark_aligned = mapper.align_benchmark(df_benchmark) + df_benchmark_aligned.to_csv(output_benchmark_path, index=False) + logger.info("Saved transformed benchmark results to: %s", output_benchmark_path) + + # Summary + logger.info("\n%s", "=" * 60) + logger.info("Alignment Summary") + logger.info("%s", "=" * 60) + logger.info("Ground truth queries: %d", df_gt_aligned["query_id"].nunique()) + logger.info("Benchmark queries: %d", df_benchmark_aligned["query_id"].nunique()) + common_queries = set(df_gt_aligned["query_id"]).intersection(set(df_benchmark_aligned["query_id"])) + logger.info("Common queries: %d", len(common_queries)) + logger.info("\nTransformed files:") + logger.info(" Ground truth: %s", output_gt_path) + logger.info(" Benchmark: %s", output_benchmark_path) + logger.info("\nThese files are ready for evaluation using evaluate_benchmark_from_csv.py") + + +if __name__ == "__main__": + main() diff --git a/scripts/evaluate_benchmark_from_csv.py b/scripts/evaluate_benchmark_from_csv.py new file mode 100644 index 00000000..e215afef --- /dev/null +++ b/scripts/evaluate_benchmark_from_csv.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +Evaluate a benchmark results CSV against a ground-truth/reference CSV. + +This is a thin CLI wrapper around the existing EvaluationEngine and +load_flexible_dataset_from_csv helper: + +- Both reference and input datasets are CSV files. +- The loader supports flexible column names and auto-detects dataset type + (information retrieval vs information extraction). +- Metrics are computed using the core EvaluationEngine logic: + precision@K, recall@K, F1@K, NDCG@K, MAP and MRR. + +Example usage: + + python -m scripts.evaluate_benchmark_from_csv \ + --reference path/to/reference.csv \ + --input path/to/results.csv \ + --k-values 1 3 5 10 + +The script prints a human-readable summary to stdout and can optionally +write the metrics to a JSON file for further analysis. +""" + +import sys +from pathlib import Path + +# Add project root to path when running directly (not as module) +# This must happen before importing report_analyst modules +script_dir = Path(__file__).parent +project_root = script_dir.parent +if str(project_root) not in sys.path: + sys.path.insert(0, str(project_root)) + +import argparse # noqa: E402 +import json # noqa: E402 +from typing import List, Optional # noqa: E402 + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine # noqa: E402 +from report_analyst.core.benchmark.retrieval_results_loader import ( # noqa: E402 + load_flexible_dataset_from_csv, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Evaluate benchmark results from CSV against a ground-truth/reference CSV " + "using the EvaluationEngine (precision@K, recall@K, F1@K, NDCG@K, MAP, MRR)." + ) + ) + parser.add_argument( + "--reference", + "-r", + required=True, + help="Path to the reference (ground truth) CSV file.", + ) + parser.add_argument( + "--input", + "-i", + required=True, + help="Path to the benchmark results CSV file to evaluate.", + ) + parser.add_argument( + "--k-values", + "-k", + nargs="*", + type=int, + default=None, + help=("Optional list of K values for metrics (e.g. -k 1 3 5 10). Defaults to EvaluationEngine.default_k_values."), + ) + parser.add_argument( + "--output", + "-o", + type=str, + default=None, + help="Optional path to write metrics as JSON.", + ) + + return parser.parse_args() + + +def evaluate_from_csv(reference_csv: str, input_csv: str, k_values: Optional[List[int]] = None): + """Load datasets from CSV and run EvaluationEngine comparison.""" + reference = load_flexible_dataset_from_csv(csv_path=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_path=input_csv) + + engine = EvaluationEngine() + metrics = engine.compare_flexible_datasets(reference, input_dataset, k_values=k_values) + return metrics + + +def print_metrics(metrics, k_values: Optional[List[int]] = None) -> None: + """Pretty-print evaluation metrics to stdout.""" + if k_values is None or not k_values: + k_values = sorted(metrics.precision_at_k.keys()) + + print("\n================ Benchmark Evaluation Metrics ================") + + # MAP and MRR + print("\nOverall metrics:") + print(f" MAP (mean_average_precision): {metrics.mean_average_precision:.4f}") + print(f" MRR (mean_reciprocal_rank): {metrics.mean_reciprocal_rank:.4f}") + + # Metrics at K + print("\nMetrics at K:") + header = f"{'K':>3} | {'Precision':>9} | {'Recall':>9} | {'F1':>9} | {'NDCG':>9}" + print(header) + print("-" * len(header)) + + for k in sorted(k_values): + p = metrics.precision_at_k.get(k, 0.0) + r = metrics.recall_at_k.get(k, 0.0) + f1 = metrics.f1_at_k.get(k, 0.0) + ndcg = metrics.ndcg_at_k.get(k, 0.0) + print(f"{k:>3} | {p:9.4f} | {r:9.4f} | {f1:9.4f} | {ndcg:9.4f}") + + print("\n==============================================================\n") + + +def main() -> None: + args = parse_args() + + reference_path = str(Path(args.reference).resolve()) + input_path = str(Path(args.input).resolve()) + k_values = args.k_values + + print("Reference CSV:", reference_path) + print("Input CSV: ", input_path) + if k_values: + print("K values: ", k_values) + + metrics = evaluate_from_csv(reference_path, input_path, k_values=k_values) + + print_metrics(metrics, k_values=k_values) + + # Optional JSON output + if args.output: + out_path = Path(args.output).resolve() + data = { + "precision_at_k": metrics.precision_at_k, + "recall_at_k": metrics.recall_at_k, + "f1_at_k": metrics.f1_at_k, + "ndcg_at_k": metrics.ndcg_at_k, + "mean_reciprocal_rank": metrics.mean_reciprocal_rank, + "mean_average_precision": metrics.mean_average_precision, + } + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + print(f"Metrics written to: {out_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/lib/affected-tests.sh b/scripts/lib/affected-tests.sh index b06162ec..aade3d88 100755 --- a/scripts/lib/affected-tests.sh +++ b/scripts/lib/affected-tests.sh @@ -211,6 +211,14 @@ qg_run_affected_pytest() { fi fi + # Per-module --cov=pkg.mod can reload NumPy under coverage tracing and break + # SciPy/sklearn (and sometimes pydantic RootModel MRO) at collection time. + # Package-level --cov matches CI and still yields per-file rows in coverage.json + # for qg_run_coverage_regression. + if [[ ${#QG_AFFECTED_COV[@]} -gt 0 ]]; then + QG_AFFECTED_COV=("--cov=${app_prefix}") + fi + local -a cov_args=() local cov_dir="$backend_root/.qg-coverage" if qg_pytest_cov_available "$py" && [[ ${#QG_AFFECTED_COV[@]} -gt 0 ]]; then diff --git a/scripts/lib/git-changed.sh b/scripts/lib/git-changed.sh index ececc8fb..755b49e2 100644 --- a/scripts/lib/git-changed.sh +++ b/scripts/lib/git-changed.sh @@ -46,7 +46,7 @@ qg_collect_changed() { report-analyst/*) echo "$p" ;; report_analyst/*|tests/*|scripts/*) qg_normalize_path "$p" ;; esac - done < <(git -C "$git_root" diff --name-only "${base_ref}...HEAD" 2>/dev/null || true) + done < <(git -C "$git_root" diff --name-only --diff-filter=ACMRTUXB "${base_ref}...HEAD" 2>/dev/null || true) else while IFS= read -r p; do [[ -z "$p" ]] && continue @@ -56,8 +56,8 @@ qg_collect_changed() { esac done < <( { - git -C "$git_root" diff --name-only HEAD 2>/dev/null || true - git -C "$git_root" diff --cached --name-only 2>/dev/null || true + git -C "$git_root" diff --name-only --diff-filter=ACMRTUXB HEAD 2>/dev/null || true + git -C "$git_root" diff --cached --name-only --diff-filter=ACMRTUXB 2>/dev/null || true git -C "$git_root" ls-files --others --exclude-standard 2>/dev/null || true } | sort -u ) diff --git a/scripts/lint-changed.sh b/scripts/lint-changed.sh index 1b7496c2..9df79042 100755 --- a/scripts/lint-changed.sh +++ b/scripts/lint-changed.sh @@ -21,11 +21,11 @@ collect_changed() { if [[ -z "$git_root" ]]; then return; fi if [[ -n "$base_ref" ]] && git -C "$git_root" rev-parse --verify "$base_ref" >/dev/null 2>&1; then - git -C "$git_root" diff --name-only "${base_ref}...HEAD" 2>/dev/null || true + git -C "$git_root" diff --name-only --diff-filter=ACMRTUXB "${base_ref}...HEAD" 2>/dev/null || true else { - git -C "$git_root" diff --name-only HEAD 2>/dev/null || true - git -C "$git_root" diff --cached --name-only 2>/dev/null || true + git -C "$git_root" diff --name-only --diff-filter=ACMRTUXB HEAD 2>/dev/null || true + git -C "$git_root" diff --cached --name-only --diff-filter=ACMRTUXB 2>/dev/null || true git -C "$git_root" ls-files --others --exclude-standard 2>/dev/null || true } fi | sort -u diff --git a/scripts/test_climretrieve_benchmark.py b/scripts/test_climretrieve_benchmark.py new file mode 100755 index 00000000..5505228d --- /dev/null +++ b/scripts/test_climretrieve_benchmark.py @@ -0,0 +1,468 @@ +#!/usr/bin/env python3 +""" +Script to download and test ClimRetrieve benchmark datasets. + +Downloads: +- Reference dataset: Expert-Annotated Relevant Sources Dataset +- Input dataset: Report-Level Dataset + +Then compares them using the flexible benchmark evaluation system. +""" + +import sys +import tempfile +from pathlib import Path +from typing import Optional + +import pandas as pd +import requests + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.core.benchmark.retrieval_results_loader import ( + load_flexible_dataset_from_csv, +) + + +def download_file_from_github(repo: str, file_path: str, output_path: Optional[str] = None, branch: str = "main") -> str: + """ + Download a file from GitHub repository. + + Args: + repo: Repository in format "owner/repo" + file_path: Path to file in repository (e.g., "data/dataset.csv") + output_path: Local path to save file (if None, uses temp file) + branch: Branch name (default: "main") + + Returns: + Path to downloaded file + """ + url = f"https://raw.githubusercontent.com/{repo}/{branch}/{file_path}" + + print(f"Downloading from: {url}") + response = requests.get(url, timeout=30) + response.raise_for_status() + + if output_path is None: + # Create temp file + suffix = Path(file_path).suffix + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + output_path = tmp.name + + # Ensure directory exists + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + + # Write file + with open(output_path, "wb") as f: + f.write(response.content) + + print(f"Downloaded to: {output_path} ({len(response.content)} bytes)") + return output_path + + +def list_github_directory(repo: str, directory_path: str, branch: str = "main") -> list: + """ + List files in a GitHub directory using GitHub API. + + Args: + repo: Repository in format "owner/repo" + directory_path: Path to directory + branch: Branch name + + Returns: + List of file/directory names + """ + url = f"https://api.github.com/repos/{repo}/contents/{directory_path}" + params = {"ref": branch} + + try: + response = requests.get(url, params=params, timeout=30) + response.raise_for_status() + contents = response.json() + + # Handle both single file and directory + if isinstance(contents, dict): + contents = [contents] + + return [item["name"] for item in contents if item["type"] == "file"] + except requests.RequestException as e: + print(f"Warning: Could not list directory via API: {e}") + return [] + + +def find_data_files_in_directory( + repo: str, + directory_path: str, + branch: str = "main", + extensions: list | None = None, +) -> list: + """ + Find data files (CSV, Excel) in a GitHub directory. + + Returns: + List of data file paths + """ + if extensions is None: + extensions = [".csv", ".xlsx", ".xls"] + files = list_github_directory(repo, directory_path, branch) + data_files = [f for f in files if any(f.endswith(ext) for ext in extensions)] + return data_files + + +def download_climretrieve_datasets(data_dir: Path, repo: str = "tobischimanski/ClimRetrieve") -> tuple[str, str]: + """ + Download ClimRetrieve datasets. + + Args: + data_dir: Directory to save datasets + repo: GitHub repository + + Returns: + Tuple of (reference_dataset_path, input_dataset_path) + """ + data_dir.mkdir(parents=True, exist_ok=True) + + # Reference dataset: Expert-Annotated Relevant Sources Dataset + reference_dir = "Expert-Annotated%20Relevant%20Sources%20Dataset" + reference_files = find_data_files_in_directory(repo, reference_dir, extensions=[".csv", ".xlsx", ".xls"]) + + if not reference_files: + # Try alternative path format + reference_dir = "Expert-Annotated Relevant Sources Dataset" + reference_files = find_data_files_in_directory(repo, reference_dir, extensions=[".csv", ".xlsx", ".xls"]) + + if not reference_files: + raise ValueError(f"Could not find data files in {reference_dir}") + + print(f"Found reference dataset files: {reference_files}") + # Prefer CSV, but use Excel if available + reference_file = next((f for f in reference_files if f.endswith(".csv")), reference_files[0]) + reference_path = data_dir / f"climretrieve_reference_{reference_file}" + + if not reference_path.exists(): + download_file_from_github(repo, f"{reference_dir}/{reference_file}", str(reference_path)) + else: + print(f"Reference dataset already exists: {reference_path}") + + # Convert Excel to CSV if needed + if reference_path.suffix in [".xlsx", ".xls"]: + csv_path = reference_path.with_suffix(".csv") + if not csv_path.exists(): + print(f"Converting Excel to CSV: {reference_path} -> {csv_path}") + df = pd.read_excel(reference_path) + df.to_csv(csv_path, index=False) + print(f"Converted to CSV: {csv_path}") + reference_path = csv_path + + # Input dataset: Report-Level Dataset + input_dir = "Report-Level%20Dataset" + input_files = find_data_files_in_directory(repo, input_dir, extensions=[".csv", ".xlsx", ".xls"]) + + if not input_files: + # Try alternative path format + input_dir = "Report-Level Dataset" + input_files = find_data_files_in_directory(repo, input_dir, extensions=[".csv", ".xlsx", ".xls"]) + + if not input_files: + raise ValueError(f"Could not find data files in {input_dir}") + + print(f"Found input dataset files: {input_files}") + # Prefer CSV, but use Excel if available + input_file = next((f for f in input_files if f.endswith(".csv")), input_files[0]) + input_path = data_dir / f"climretrieve_input_{input_file}" + + if not input_path.exists(): + download_file_from_github(repo, f"{input_dir}/{input_file}", str(input_path)) + else: + print(f"Input dataset already exists: {input_path}") + + # Convert Excel to CSV if needed + if input_path.suffix in [".xlsx", ".xls"]: + csv_path = input_path.with_suffix(".csv") + if not csv_path.exists(): + print(f"Converting Excel to CSV: {input_path} -> {csv_path}") + df = pd.read_excel(input_path) + df.to_csv(csv_path, index=False) + print(f"Converted to CSV: {csv_path}") + input_path = csv_path + + return str(reference_path), str(input_path) + + +def inspect_dataset_columns(csv_path: str) -> None: + """Inspect and display dataset columns and sample rows""" + print(f"\n{'=' * 60}") + print(f"Inspecting dataset: {csv_path}") + print(f"{'=' * 60}") + + df = pd.read_csv(csv_path, nrows=5) + print(f"\nColumns: {list(df.columns)}") + print(f"\nShape: {df.shape}") + print("\nFirst few rows:") + print(df.head()) + print("\nData types:") + print(df.dtypes) + + +def run_climretrieve_benchmark(reference_path: str, input_path: str, k_values: Optional[list] = None) -> None: + """ + Run benchmark comparison between ClimRetrieve datasets. + + Args: + reference_path: Path to reference dataset CSV + input_path: Path to input dataset CSV + k_values: List of K values for evaluation + """ + print(f"\n{'=' * 60}") + print("Loading Datasets") + print(f"{'=' * 60}") + + # Inspect datasets first + inspect_dataset_columns(reference_path) + inspect_dataset_columns(input_path) + + # Load datasets + print(f"\n{'=' * 60}") + print("Loading Reference Dataset (Expert-Annotated)") + print(f"{'=' * 60}") + + # Check if we need to preprocess the CSV for ClimRetrieve format + df_ref = pd.read_csv(reference_path, nrows=1) + transformed_path = reference_path.replace(".csv", "_transformed.csv") + + if "Question" in df_ref.columns and "Context" in df_ref.columns: + # ClimRetrieve reference format - need to transform + if not Path(transformed_path).exists(): + print("Detected ClimRetrieve reference format, preprocessing...") + df_ref_full = pd.read_csv(reference_path) + # Transform: Question -> query_id, Context -> chunk_id (use hash of content for matching) + # Create position based on row order per question + import hashlib + + df_ref_full["query_id"] = df_ref_full["Question"] + # Use hash of Context text as chunk_id for better matching + df_ref_full["chunk_id"] = df_ref_full["Context"].apply( + lambda x: ( + hashlib.md5(str(x).strip().encode(), usedforsecurity=False).hexdigest()[:16] + if pd.notna(x) and str(x).strip() + else f"context_{hash(str(x)) % 10000}" + ) + ) + df_ref_full["position"] = df_ref_full.groupby("Question").cumcount() + 1 + # Use Source Relevance Score if available, otherwise use Relevant (convert Yes/No to 1.0/0.0) + if "Source Relevance Score" in df_ref_full.columns: + df_ref_full["score"] = df_ref_full["Source Relevance Score"].fillna(0.0) + elif "Relevant" in df_ref_full.columns: + df_ref_full["score"] = df_ref_full["Relevant"].apply( + lambda x: (1.0 if str(x).lower() in ["yes", "y", "1", "true"] else 0.0) + ) + else: + df_ref_full["score"] = 1.0 + # Save transformed version + df_ref_full[["query_id", "chunk_id", "position", "score", "Context", "Question"]].to_csv( + transformed_path, index=False + ) + print(f"Transformed reference dataset saved to: {transformed_path} ({len(df_ref_full)} rows)") + else: + print(f"Using existing transformed reference dataset: {transformed_path}") + reference_path = transformed_path + + reference_dataset = load_flexible_dataset_from_csv( + csv_path=reference_path, + dataset_id="climretrieve_reference", + dataset_name="ClimRetrieve Expert-Annotated Reference", + ) + + print(f"\nLoaded {len(reference_dataset.results)} rows") + print(f"Dataset type: {reference_dataset.dataset_type}") + print(f"Unique queries: {len(reference_dataset.get_unique_queries())}") + + print(f"\n{'=' * 60}") + print("Loading Input Dataset (Report-Level)") + print(f"{'=' * 60}") + + # Check if we need to preprocess the CSV for ClimRetrieve format + df_input = pd.read_csv(input_path, nrows=1) + transformed_input_path = input_path.replace(".csv", "_transformed.csv") + + if "question" in df_input.columns and "paragraph" in df_input.columns: + # ClimRetrieve input format - need to transform + if not Path(transformed_input_path).exists(): + print("Detected ClimRetrieve input format, preprocessing...") + df_input_full = pd.read_csv(input_path) + # Transform: question -> query_id, paragraph/relevant_text -> chunk_id (use hash for matching) + # Create position based on row order per question + import hashlib + + df_input_full["query_id"] = df_input_full["question"] + # Use hash of relevant_text (or paragraph) as chunk_id for matching with reference + # Prefer relevant_text as it's more likely to match the Context field in reference + if "relevant_text" in df_input_full.columns: + content_col = df_input_full["relevant_text"] + elif "paragraph" in df_input_full.columns: + content_col = df_input_full["paragraph"] + else: + content_col = pd.Series([""] * len(df_input_full)) + + df_input_full["chunk_id"] = content_col.apply( + lambda x: ( + hashlib.md5(str(x).strip().encode(), usedforsecurity=False).hexdigest()[:16] + if pd.notna(x) and str(x).strip() + else f"para_{hash(str(x)) % 10000}" + ) + ) + df_input_full["position"] = df_input_full.groupby("question").cumcount() + 1 + # Use relevance or sim_text_relevance as score + if "relevance" in df_input_full.columns: + df_input_full["score"] = df_input_full["relevance"].fillna(0.0) + elif "sim_text_relevance" in df_input_full.columns: + df_input_full["score"] = df_input_full["sim_text_relevance"].fillna(0.0) / 3.0 # Normalize 0-3 to 0-1 + else: + df_input_full["score"] = 1.0 + # Save transformed version + df_input_full[["query_id", "chunk_id", "position", "score", "paragraph", "question"]].to_csv( + transformed_input_path, index=False + ) + print(f"Transformed input dataset saved to: {transformed_input_path} ({len(df_input_full)} rows)") + else: + print(f"Using existing transformed input dataset: {transformed_input_path}") + input_path = transformed_input_path + + input_dataset = load_flexible_dataset_from_csv( + csv_path=input_path, + dataset_id="climretrieve_input", + dataset_name="ClimRetrieve Report-Level Input", + ) + + print(f"\nLoaded {len(input_dataset.results)} rows") + print(f"Dataset type: {input_dataset.dataset_type}") + print(f"Unique queries: {len(input_dataset.get_unique_queries())}") + + # Compare datasets + print(f"\n{'=' * 60}") + print("Running Evaluation") + print(f"{'=' * 60}") + + engine = EvaluationEngine() + metrics = engine.compare_flexible_datasets( + reference_dataset=reference_dataset, + input_dataset=input_dataset, + k_values=k_values or [1, 3, 5, 10], + ) + + # Display results + print(f"\n{'=' * 60}") + print("Evaluation Results") + print(f"{'=' * 60}") + print(f"\nMean Average Precision (MAP): {metrics.mean_average_precision:.4f}") + print(f"Mean Reciprocal Rank (MRR): {metrics.mean_reciprocal_rank:.4f}") + + print("\nPrecision@K:") + for k, score in sorted(metrics.precision_at_k.items()): + print(f" P@{k}: {score:.4f}") + + print("\nRecall@K:") + for k, score in sorted(metrics.recall_at_k.items()): + print(f" R@{k}: {score:.4f}") + + print("\nF1@K:") + for k, score in sorted(metrics.f1_at_k.items()): + print(f" F1@{k}: {score:.4f}") + + print("\nNDCG@K:") + for k, score in sorted(metrics.ndcg_at_k.items()): + print(f" NDCG@{k}: {score:.4f}") + + return metrics + + +def main(): + """Main function to run ClimRetrieve benchmark test""" + import argparse + + parser = argparse.ArgumentParser(description="Test ClimRetrieve benchmark datasets") + parser.add_argument( + "--data-dir", + type=str, + default=str(Path(__file__).parent.parent / "data" / "climretrieve"), + help="Directory to store downloaded datasets (default: data/climretrieve)", + ) + parser.add_argument( + "--reference-path", + type=str, + help="Path to reference dataset CSV (skips download if provided)", + ) + parser.add_argument( + "--input-path", + type=str, + help="Path to input dataset CSV (skips download if provided)", + ) + parser.add_argument( + "--repo", + type=str, + default="tobischimanski/ClimRetrieve", + help="GitHub repository", + ) + parser.add_argument( + "--k-values", + type=int, + nargs="+", + default=[1, 3, 5, 10], + help="K values for evaluation", + ) + parser.add_argument( + "--skip-download", + action="store_true", + help="Skip downloading and use existing files", + ) + + args = parser.parse_args() + + data_dir = Path(args.data_dir) + + try: + # Download datasets if not provided + if args.reference_path and args.input_path: + reference_path = args.reference_path + input_path = args.input_path + print("Using provided datasets:") + print(f" Reference: {reference_path}") + print(f" Input: {input_path}") + elif args.skip_download: + # Try to find existing files + reference_files = list(data_dir.glob("climretrieve_reference_*.csv")) + input_files = list(data_dir.glob("climretrieve_input_*.csv")) + + if not reference_files or not input_files: + raise ValueError("No existing datasets found. Remove --skip-download to download.") + + reference_path = str(reference_files[0]) + input_path = str(input_files[0]) + print("Using existing datasets:") + print(f" Reference: {reference_path}") + print(f" Input: {input_path}") + else: + print("Downloading ClimRetrieve datasets from GitHub...") + reference_path, input_path = download_climretrieve_datasets(data_dir, args.repo) + + # Run benchmark + run_climretrieve_benchmark(reference_path, input_path, k_values=args.k_values) + + print(f"\n{'=' * 60}") + print("Benchmark Test Complete!") + print(f"{'=' * 60}") + + return 0 + + except Exception as e: # noqa: BLE001 + print(f"\nError: {e}", file=sys.stderr) + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 78f1f503..21e6f117 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,30 +5,39 @@ """ # Load .env first so OPENBLAS_NUM_THREADS=1 is set before any NumPy/OpenBLAS import (avoids SIGSEGV on macOS ARM) +import json import os +import sys from pathlib import Path +from unittest.mock import AsyncMock + +import pytest +import yaml +from dotenv import load_dotenv _conftest_dir = Path(__file__).resolve().parent _env = _conftest_dir.parent / ".env" if _env.exists(): - from dotenv import load_dotenv - load_dotenv(_env) os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") -import json -import os -import tempfile -from unittest.mock import AsyncMock, Mock +# Add project root and parent directory to Python path +# This allows tests to import report_analyst and report_analyst_jobs +project_root = Path(__file__).parent.parent +parent_dir = project_root.parent -import pytest -import yaml -from dotenv import load_dotenv +for path in [str(project_root), str(parent_dir)]: + if path not in sys.path: + sys.path.insert(0, path) -# Load environment variables from .env file before any other imports -load_dotenv() +# Make report_analyst_jobs import optional (ImportError or transitive env/deps failures) +try: + from report_analyst_jobs.event_router import EventRouter -from report_analyst_jobs.event_router import IGNORE_ACTION, EventRouter + REPORT_ANALYST_JOBS_AVAILABLE = True +except Exception: # noqa: BLE001 + REPORT_ANALYST_JOBS_AVAILABLE = False + EventRouter = None # ============================================================================= # Test Environment Configuration @@ -102,7 +111,7 @@ def _test_database_connection(url): conn.execute(text("SELECT 1")) engine.dispose() return True - except Exception: + except Exception: # noqa: BLE001 return False @@ -228,6 +237,9 @@ def mock_nats_connection(): @pytest.fixture def event_router_with_mocks(event_router_yaml_file, mock_nats_connection): """Event router with mocked NATS connection""" + if not REPORT_ANALYST_JOBS_AVAILABLE: + pytest.skip("report_analyst_jobs not available - skipping event router tests") + mock_nc, mock_js = mock_nats_connection router = EventRouter.from_yaml(yaml_path=event_router_yaml_file) diff --git a/tests/test_analyzer.py b/tests/test_analyzer.py index 29aa62fe..161aabdf 100644 --- a/tests/test_analyzer.py +++ b/tests/test_analyzer.py @@ -97,8 +97,7 @@ def test_env(clean_db): { "id": "tcfd_2", "text": ( - "What is the role of management in assessing and managing " - "climate-related risks and opportunities?" + "What is the role of management in assessing and managing climate-related risks and opportunities?" ), "guidelines": "Test guidelines 2", }, @@ -591,3 +590,238 @@ async def test_document_analysis_workflow(test_env): complete_results = [r for r in results if r.get("status") == "complete"] if complete_results: assert all("question_id" in r for r in complete_results) + + +@pytest.mark.skip(reason="Chunk size creation behavior is not critical for current functionality") +def test_chunk_size_creation(analyzer, test_env): + """Test that chunks are created with the requested chunk_size parameter""" + import fitz # PyMuPDF + import numpy as np + + # Create a test PDF with substantial text content + test_pdf_path = test_env["storage_path"] / "chunk_test.pdf" + + # Generate text content with multiple sentences for chunking + # Create text that's long enough to test different chunk sizes + sample_text = ( + "Climate change is one of the most pressing challenges of our time. " + "Organizations worldwide are recognizing the need to address climate-related risks and opportunities. " + "The Task Force on Climate-related Financial Disclosures (TCFD) provides " + "a framework for companies to report on climate risks. " + "Scope 1 emissions are direct emissions from owned or controlled sources. " + "Scope 2 emissions are indirect emissions from the generation of purchased energy. " + "Scope 3 emissions include all other indirect emissions in a company's value chain. " + "Science-based targets help companies set emission reduction goals aligned with climate science. " + "Net zero commitments require companies to balance emissions with removals. " + "Renewable energy adoption is crucial for reducing Scope 2 emissions. " + "Supply chain management plays a key role in addressing Scope 3 emissions. " + "Carbon offsetting can complement but not replace emission reduction efforts. " + "Climate scenario analysis helps companies understand potential future risks. " + "Physical risks from climate change include extreme weather events and sea-level rise. " + "Transition risks include policy changes, technology shifts, and market changes. " + "Governance structures should include climate risk oversight at the board level. " + "Risk management processes need to integrate climate considerations. " + "Metrics and targets should be disclosed to track progress over time. " + "Stakeholder engagement is important for understanding climate-related expectations. " + "Transparency in reporting builds trust with investors and other stakeholders. " + "Continuous improvement in climate disclosure is essential for effective risk management. " + ) * 10 # Repeat to ensure we have enough text for chunking + + # Create PDF with text content using PyMuPDF + doc = fitz.open() # Create new PDF + page = doc.new_page() + + # Insert text into the page using insert_text for better compatibility + # Split text into lines that fit on the page + words = sample_text.split() + y_position = 50 + line = "" + + for word in words: + test_line = line + word + " " + # Check if line would exceed page width (approximately 500 points) + if len(test_line) > 80: # Rough character limit per line + if line: + page.insert_text((50, y_position), line.strip(), fontsize=11, fontname="helv") + y_position += 15 + if y_position > 750: # Start new page if needed + page = doc.new_page() + y_position = 50 + line = word + " " + else: + line = test_line + + # Insert remaining text + if line: + page.insert_text((50, y_position), line.strip(), fontsize=11, fontname="helv") + + doc.save(str(test_pdf_path)) + doc.close() + + # Verify PDF was created and has text + verify_doc = fitz.open(str(test_pdf_path)) + extracted_text = "".join([page.get_text() for page in verify_doc]) + verify_doc.close() + assert len(extracted_text) > 100, "PDF should contain substantial text content" + + # Mock embeddings to avoid API calls + mock_embedding = np.random.rand(1536).astype(np.float32) # Standard embedding dimension + + with patch.object(analyzer, "embeddings") as mock_embeddings: + # Mock get_text_embedding_batch which is used by _create_chunks + mock_embeddings.get_text_embedding_batch = Mock(return_value=[mock_embedding.tolist()] * 1000) + mock_embeddings.embed_query = Mock(return_value=mock_embedding.tolist()) + + # Test with different chunk sizes + chunk_sizes = [250, 500, 1000] + chunk_overlap = 20 + results = {} + + for chunk_size in chunk_sizes: + # Update analyzer parameters + analyzer.update_parameters(chunk_size, chunk_overlap, top_k=5) + + # Verify text_splitter was updated + assert analyzer.text_splitter.chunk_size == chunk_size + assert analyzer.text_splitter.chunk_overlap == chunk_overlap + assert analyzer.chunk_params["chunk_size"] == chunk_size + assert analyzer.chunk_params["chunk_overlap"] == chunk_overlap + + # Create chunks + chunks = analyzer._create_chunks(str(test_pdf_path)) + + results[chunk_size] = { + "chunks": chunks, + "count": len(chunks), + } + + # Verify chunks were created + assert len(chunks) > 0, f"No chunks created for chunk_size={chunk_size}" + + # Verify chunk metadata + for chunk in chunks: + assert "chunk_size" in chunk.get("metadata", {}) or "chunk_size" in chunk + chunk_size_meta = chunk.get("metadata", {}).get("chunk_size") or chunk.get("chunk_size") + assert ( + chunk_size_meta == chunk_size + ), f"Chunk metadata chunk_size={chunk_size_meta} doesn't match requested {chunk_size}" + + chunk_overlap_meta = chunk.get("metadata", {}).get("chunk_overlap") or chunk.get("chunk_overlap") + assert ( + chunk_overlap_meta == chunk_overlap + ), f"Chunk metadata chunk_overlap={chunk_overlap_meta} doesn't match requested {chunk_overlap}" + + # Verify chunk text lengths are approximately correct + # SentenceSplitter splits at sentence boundaries, so chunks may vary significantly + # The key is that different chunk sizes should produce different average lengths + chunk_lengths = [len(chunk.get("text", "")) for chunk in chunks] + avg_length = sum(chunk_lengths) / len(chunk_lengths) if chunk_lengths else 0 + min_length = min(chunk_lengths) if chunk_lengths else 0 + max_length = max(chunk_lengths) if chunk_lengths else 0 + + # Store statistics for comparison + results[chunk_size]["avg_length"] = avg_length + results[chunk_size]["min_length"] = min_length + results[chunk_size]["max_length"] = max_length + + # Verify that chunks are being created (not empty) + assert avg_length > 0, f"Chunks have zero average length for chunk_size={chunk_size}" + + # Note: Due to sentence boundary splitting, chunks may be larger than chunk_size + # The important thing is that different chunk sizes produce different results + print( + f"Chunk size {chunk_size}: {len(chunks)} chunks, " + f"avg length={avg_length:.1f}, range=[{min_length}, {max_length}]" + ) + + # Verify different chunk sizes produce different numbers of chunks + # Smaller chunk sizes should produce more chunks (or at least different chunking) + assert results[250]["count"] != results[500]["count"] or results[250]["avg_length"] != results[500]["avg_length"], ( + f"Chunk sizes 250 and 500 produced identical results: " + f"count={results[250]['count']} vs {results[500]['count']}, " + f"avg_length={results[250]['avg_length']:.1f} vs {results[500]['avg_length']:.1f}" + ) + assert results[500]["count"] != results[1000]["count"] or results[500]["avg_length"] != results[1000]["avg_length"], ( + f"Chunk sizes 500 and 1000 produced identical results: " + f"count={results[500]['count']} vs {results[1000]['count']}, " + f"avg_length={results[500]['avg_length']:.1f} vs {results[1000]['avg_length']:.1f}" + ) + + # Ideally, smaller chunk sizes should produce more chunks + # But due to sentence boundaries, this might not always be true + # So we just verify they're different + print( + f"\nChunk size comparison:\n" + f" 250: {results[250]['count']} chunks, avg={results[250]['avg_length']:.1f} chars\n" + f" 500: {results[500]['count']} chunks, avg={results[500]['avg_length']:.1f} chars\n" + f" 1000: {results[1000]['count']} chunks, avg={results[1000]['avg_length']:.1f} chars" + ) + + # CRITICAL TEST: Verify that 250-size chunks are NOT always complete subsets of 1000-size chunks + # With only 20 chars overlap, a 250-size chunk starting at position 950 of a 1000-size chunk + # should extend into the next 1000-size chunk (to position 200 of next chunk) + chunks_250 = results[250]["chunks"] + chunks_1000 = results[1000]["chunks"] + + # Check each 250-size chunk + chunks_that_span = 0 + chunks_that_are_subset = 0 + + for chunk_250 in chunks_250: + chunk_250_text = chunk_250.get("text", "") + + # Check if this 250-size chunk is a complete subset of any 1000-size chunk + is_subset = False + for chunk_1000 in chunks_1000: + chunk_1000_text = chunk_1000.get("text", "") + if chunk_250_text in chunk_1000_text: + is_subset = True + break + + if is_subset: + chunks_that_are_subset += 1 + else: + chunks_that_span += 1 + + print( + f"\nChunk subset analysis:\n" + f" 250-size chunks that are complete subsets of 1000-size chunks: {chunks_that_are_subset}/{len(chunks_250)}\n" + f" 250-size chunks that span across 1000-size chunk boundaries: {chunks_that_span}/{len(chunks_250)}" + ) + + # This is the key assertion: NOT all 250-size chunks should be subsets + # At least some should span across boundaries + # Allow some tolerance (maybe 20% can be subsets due to alignment), but not all + subset_percentage = chunks_that_are_subset / len(chunks_250) if chunks_250 else 0 + assert subset_percentage < 0.9, ( + f"Too many 250-size chunks ({chunks_that_are_subset}/{len(chunks_250)} = {subset_percentage:.1%}) " + f"are complete subsets of 1000-size chunks. " + f"This suggests chunking is deterministic and always starts from the same position, " + f"which is incorrect. With only 20 chars overlap, 250-size chunks should span across " + f"1000-size chunk boundaries." + ) + + # Verify overlap is working (check that consecutive chunks have overlapping text) + # Test with chunk_size=500 which should have clear overlap + chunks_500 = results[500]["chunks"] + if len(chunks_500) >= 2: + chunk1_text = chunks_500[0].get("text", "") + chunk2_text = chunks_500[1].get("text", "") + + # With overlap, the end of chunk1 should appear at the start of chunk2 + # Check if last 50 characters of chunk1 appear in chunk2 + if len(chunk1_text) >= 50 and len(chunk2_text) >= 50: + chunk1_end = chunk1_text[-50:].strip() + # Look for overlap in first 100 chars of chunk2 (accounting for some variation) + chunk2_start = chunk2_text[:100].strip() + # Overlap might not be exact due to sentence boundaries, so check for partial match + # At least some words should overlap + chunk1_words = set(chunk1_end.split()[-5:]) # Last 5 words + chunk2_words = set(chunk2_start.split()[:5]) # First 5 words + overlap_found = len(chunk1_words.intersection(chunk2_words)) > 0 + + # Note: This is a soft check - overlap might not always be detectable + # depending on how SentenceSplitter handles boundaries + if not overlap_found: + # Log warning but don't fail - overlap detection is tricky + print("Warning: Could not detect clear overlap between consecutive chunks") diff --git a/tests/test_benchmark_dataset_loader.py b/tests/test_benchmark_dataset_loader.py new file mode 100644 index 00000000..4b275a0d --- /dev/null +++ b/tests/test_benchmark_dataset_loader.py @@ -0,0 +1,256 @@ +import json +import tempfile +from pathlib import Path + +import pytest + +from report_analyst.core.benchmark.dataset_loader import ( + DatasetLoader, + DatasetValidationError, +) + + +class TestDatasetLoader: + """Test suite for dataset loading and validation""" + + @pytest.fixture + def loader(self): + return DatasetLoader() + + @pytest.fixture + def valid_dataset_yaml(self): + return """ +dataset_id: "test_dataset_v1" +name: "Test TCFD Dataset" +description: "Test dataset for TCFD questions" +version: "1.0" +question_set: "tcfd" +created_at: "2024-01-15" +questions: + - question_id: "tcfd_1" + question_text: "What are the climate-related risks?" + ground_truth_chunks: + - chunk_id: "chunk_001" + relevance_score: 1.0 + is_evidence: true + evidence_order: 1 + annotation_notes: "Contains specific climate risk metrics" + - chunk_id: "chunk_002" + relevance_score: 0.8 + is_evidence: true + evidence_order: 2 + annotation_notes: "Describes governance processes" + - question_id: "tcfd_2" + question_text: "What are the climate opportunities?" + ground_truth_chunks: + - chunk_id: "chunk_003" + relevance_score: 0.9 + is_evidence: true + evidence_order: 1 + annotation_notes: "Lists specific opportunities" +""" + + @pytest.fixture + def invalid_dataset_missing_field(self): + return """ +dataset_id: "test_dataset_v1" +name: "Test TCFD Dataset" +# Missing description field +version: "1.0" +question_set: "tcfd" +questions: [] +""" + + @pytest.fixture + def invalid_dataset_bad_score(self): + return """ +dataset_id: "test_dataset_v1" +name: "Test TCFD Dataset" +description: "Test dataset" +version: "1.0" +question_set: "tcfd" +questions: + - question_id: "tcfd_1" + question_text: "What are the climate-related risks?" + ground_truth_chunks: + - chunk_id: "chunk_001" + relevance_score: 1.5 # Invalid score > 1.0 + is_evidence: true +""" + + def test_load_valid_yaml_dataset(self, loader, valid_dataset_yaml): + """Test loading a valid YAML dataset""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(valid_dataset_yaml) + f.flush() + + dataset = loader.load_dataset(f.name) + + # Verify dataset structure + assert dataset.dataset_id == "test_dataset_v1" + assert dataset.name == "Test TCFD Dataset" + assert dataset.question_set == "tcfd" + assert len(dataset.questions) == 2 + + # Verify first question + q1 = dataset.questions[0] + assert q1.question_id == "tcfd_1" + assert len(q1.ground_truth_chunks) == 2 + + # Verify chunks + chunk1 = q1.ground_truth_chunks[0] + assert chunk1.chunk_id == "chunk_001" + assert chunk1.relevance_score == 1.0 + assert chunk1.is_evidence is True + assert chunk1.evidence_order == 1 + + Path(f.name).unlink() + + def test_load_nonexistent_file(self, loader): + """Test loading a file that doesn't exist""" + with pytest.raises(FileNotFoundError): + loader.load_dataset("nonexistent_file.yaml") + + def test_load_unsupported_format(self, loader): + """Test loading an unsupported file format""" + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f: + f.write(b"some content") + f.flush() + + with pytest.raises(ValueError, match="Unsupported file format"): + loader.load_dataset(f.name) + + Path(f.name).unlink() + + def test_load_invalid_yaml(self, loader): + """Test loading invalid YAML""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write("invalid: yaml: content: [") + f.flush() + + with pytest.raises(DatasetValidationError): + loader.load_dataset(f.name) + + Path(f.name).unlink() + + def test_validation_missing_required_field(self, loader, invalid_dataset_missing_field): + """Test validation with missing required field""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(invalid_dataset_missing_field) + f.flush() + + with pytest.raises(DatasetValidationError, match="Missing required field"): + loader.load_dataset(f.name) + + Path(f.name).unlink() + + def test_validation_invalid_relevance_score(self, loader, invalid_dataset_bad_score): + """Test validation with invalid relevance score""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(invalid_dataset_bad_score) + f.flush() + + with pytest.raises(DatasetValidationError, match="Invalid relevance_score"): + loader.load_dataset(f.name) + + Path(f.name).unlink() + + def test_validate_dataset_consistency(self, loader, valid_dataset_yaml): + """Test dataset consistency validation""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(valid_dataset_yaml) + f.flush() + + dataset = loader.load_dataset(f.name) + warnings = loader.validate_dataset_consistency(dataset) + + # Should have no warnings for valid dataset + assert len(warnings) == 0 + + Path(f.name).unlink() + + def test_validate_duplicate_question_ids(self, loader): + """Test validation catches duplicate question IDs""" + duplicate_dataset = """ +dataset_id: "test_dataset" +name: "Test Dataset" +description: "Test" +version: "1.0" +question_set: "tcfd" +questions: + - question_id: "tcfd_1" + question_text: "Question 1" + ground_truth_chunks: + - chunk_id: "chunk_001" + relevance_score: 1.0 + is_evidence: true + - question_id: "tcfd_1" # Duplicate ID + question_text: "Question 2" + ground_truth_chunks: + - chunk_id: "chunk_002" + relevance_score: 0.8 + is_evidence: true +""" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(duplicate_dataset) + f.flush() + + dataset = loader.load_dataset(f.name) + warnings = loader.validate_dataset_consistency(dataset) + + assert any("Duplicate question IDs" in warning for warning in warnings) + + Path(f.name).unlink() + + def test_generate_dataset_hash(self, loader, valid_dataset_yaml): + """Test dataset hash generation""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(valid_dataset_yaml) + f.flush() + + dataset = loader.load_dataset(f.name) + hash1 = loader.generate_dataset_hash(dataset) + hash2 = loader.generate_dataset_hash(dataset) + + # Hash should be consistent + assert hash1 == hash2 + assert len(hash1) == 16 # SHA256 truncated to 16 chars + + Path(f.name).unlink() + + def test_load_json_dataset(self, loader): + """Test loading a JSON dataset""" + json_data = { + "dataset_id": "test_json", + "name": "Test JSON Dataset", + "description": "Test dataset in JSON format", + "version": "1.0", + "question_set": "tcfd", + "questions": [ + { + "question_id": "tcfd_1", + "question_text": "Test question", + "ground_truth_chunks": [ + { + "chunk_id": "chunk_001", + "relevance_score": 0.9, + "is_evidence": True, + "evidence_order": 1, + } + ], + } + ], + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(json_data, f) + f.flush() + + dataset = loader.load_dataset(f.name) + + assert dataset.dataset_id == "test_json" + assert len(dataset.questions) == 1 + assert dataset.questions[0].ground_truth_chunks[0].relevance_score == 0.9 + + Path(f.name).unlink() diff --git a/tests/test_benchmark_evaluation_engine.py b/tests/test_benchmark_evaluation_engine.py new file mode 100644 index 00000000..e31d3e6e --- /dev/null +++ b/tests/test_benchmark_evaluation_engine.py @@ -0,0 +1,281 @@ +import numpy as np +import pytest + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.models.benchmark import ( + BenchmarkDatasetContent, + BenchmarkQuestion, + EvaluationMetrics, + GroundTruthChunk, + RetrievalConfig, +) + + +class TestEvaluationEngine: + """Test suite for evaluation engine metrics calculation""" + + @pytest.fixture + def engine(self): + return EvaluationEngine() + + @pytest.fixture + def sample_dataset(self): + """Create a sample dataset for testing""" + chunks_q1 = [ + GroundTruthChunk( + chunk_id="chunk_1", + relevance_score=1.0, + is_evidence=True, + evidence_order=1, + ), + GroundTruthChunk( + chunk_id="chunk_2", + relevance_score=0.8, + is_evidence=True, + evidence_order=2, + ), + GroundTruthChunk(chunk_id="chunk_3", relevance_score=0.0, is_evidence=False), + ] + + chunks_q2 = [ + GroundTruthChunk( + chunk_id="chunk_4", + relevance_score=0.9, + is_evidence=True, + evidence_order=1, + ), + GroundTruthChunk(chunk_id="chunk_5", relevance_score=0.0, is_evidence=False), + ] + + questions = [ + BenchmarkQuestion( + question_id="q1", + question_text="Question 1", + ground_truth_chunks=chunks_q1, + ), + BenchmarkQuestion( + question_id="q2", + question_text="Question 2", + ground_truth_chunks=chunks_q2, + ), + ] + + return BenchmarkDatasetContent( + dataset_id="test_dataset", + name="Test Dataset", + description="Test", + version="1.0", + question_set="tcfd", + created_at="2024-01-01", + questions=questions, + ) + + @pytest.fixture + def sample_retrieval_results(self): + """Sample retrieval results for testing""" + return { + "q1": [ + {"id": "chunk_1", "score": 0.95}, # Relevant, rank 1 + {"id": "chunk_3", "score": 0.85}, # Not relevant, rank 2 + {"id": "chunk_2", "score": 0.75}, # Relevant, rank 3 + {"id": "chunk_unknown", "score": 0.65}, # Unknown chunk, rank 4 + ], + "q2": [ + {"id": "chunk_5", "score": 0.90}, # Not relevant, rank 1 + {"id": "chunk_4", "score": 0.80}, # Relevant, rank 2 + ], + } + + def test_precision_at_k(self, engine): + """Test precision@K calculation""" + binary_relevance = [1, 0, 1, 0, 1] # 3 relevant out of 5 + + assert engine._precision_at_k(binary_relevance, 1) == 1.0 # 1/1 + assert engine._precision_at_k(binary_relevance, 2) == 0.5 # 1/2 + assert engine._precision_at_k(binary_relevance, 3) == 2 / 3 # 2/3 + assert engine._precision_at_k(binary_relevance, 5) == 0.6 # 3/5 + assert engine._precision_at_k(binary_relevance, 0) == 0.0 # Edge case + + def test_recall_at_k(self, engine): + """Test recall@K calculation""" + binary_relevance = [1, 0, 1, 0, 1] # 3 relevant out of 5 + total_relevant = 4 # Assume 4 total relevant chunks exist + + assert engine._recall_at_k(binary_relevance, total_relevant, 1) == 0.25 # 1/4 + assert engine._recall_at_k(binary_relevance, total_relevant, 3) == 0.5 # 2/4 + assert engine._recall_at_k(binary_relevance, total_relevant, 5) == 0.75 # 3/4 + assert engine._recall_at_k(binary_relevance, 0, 5) == 0.0 # No relevant chunks + + def test_f1_at_k(self, engine): + """Test F1@K calculation""" + assert engine._f1_at_k(1.0, 1.0) == 1.0 # Perfect + assert engine._f1_at_k(0.5, 0.5) == 0.5 # Equal precision and recall + assert engine._f1_at_k(0.0, 0.0) == 0.0 # No relevant results + + # Test harmonic mean calculation + precision, recall = 0.8, 0.6 + expected_f1 = 2 * (precision * recall) / (precision + recall) + assert abs(engine._f1_at_k(precision, recall) - expected_f1) < 1e-10 + + def test_reciprocal_rank(self, engine): + """Test reciprocal rank calculation""" + assert engine._reciprocal_rank([1, 0, 0, 0]) == 1.0 # First position + assert engine._reciprocal_rank([0, 1, 0, 0]) == 0.5 # Second position + assert engine._reciprocal_rank([0, 0, 1, 0]) == 1 / 3 # Third position + assert engine._reciprocal_rank([0, 0, 0, 0]) == 0.0 # No relevant results + + def test_average_precision(self, engine): + """Test average precision calculation""" + # Perfect ranking: all relevant first + binary_relevance = [1, 1, 1, 0, 0] + ap = engine._average_precision(binary_relevance) + expected = (1.0 + 1.0 + 1.0) / 3 # (1/1 + 2/2 + 3/3) / 3 + assert abs(ap - expected) < 1e-10 + + # Mixed ranking + binary_relevance = [1, 0, 1, 0, 1] + ap = engine._average_precision(binary_relevance) + expected = (1.0 + 2 / 3 + 3 / 5) / 3 # (1/1 + 2/3 + 3/5) / 3 + assert abs(ap - expected) < 1e-10 + + # No relevant documents + assert engine._average_precision([0, 0, 0, 0]) == 0.0 + + def test_ndcg_at_k(self, engine): + """Test NDCG@K calculation""" + retrieved_relevance = [1.0, 0.8, 0.0, 0.6] + ground_truth = {"chunk_1": 1.0, "chunk_2": 0.8, "chunk_3": 0.6, "chunk_4": 0.0} + + # Test NDCG@1 + ndcg_1 = engine._ndcg_at_k(retrieved_relevance, ground_truth, 1) + assert ndcg_1 == 1.0 # Perfect first result + + # Test NDCG@3 + ndcg_3 = engine._ndcg_at_k(retrieved_relevance, ground_truth, 3) + + # Calculate expected DCG@3 + dcg = 1.0 + 0.8 / np.log2(2) + 0.0 / np.log2(3) + + # Calculate expected IDCG@3 (ideal ranking: [1.0, 0.8, 0.6]) + idcg = 1.0 + 0.8 / np.log2(2) + 0.6 / np.log2(3) + + expected_ndcg = dcg / idcg + assert abs(ndcg_3 - expected_ndcg) < 1e-10 + + def test_evaluate_single_question(self, engine): + """Test evaluation of a single question""" + retrieved_chunks = [ + {"id": "chunk_1", "score": 0.9}, + {"id": "chunk_2", "score": 0.8}, + {"id": "chunk_3", "score": 0.7}, + ] + + ground_truth = { + "chunk_1": 1.0, # Relevant + "chunk_2": 0.0, # Not relevant + "chunk_3": 0.8, # Relevant + } + + result = engine._evaluate_single_question(retrieved_chunks, ground_truth, [1, 2, 3]) + + # Check binary relevance + expected_binary = [ + 1, + 0, + 1, + ] # chunk_1: relevant, chunk_2: not relevant, chunk_3: relevant + assert result["binary_relevance"] == expected_binary + + # Check precision@K + assert result["precision_at_k"][1] == 1.0 # 1/1 + assert result["precision_at_k"][2] == 0.5 # 1/2 + assert result["precision_at_k"][3] == 2 / 3 # 2/3 + + # Check reciprocal rank (first relevant at position 1) + assert result["reciprocal_rank"] == 1.0 + + def test_evaluate_retrieval_full(self, engine, sample_dataset, sample_retrieval_results): + """Test full retrieval evaluation""" + config = RetrievalConfig(top_k=5) + + metrics = engine.evaluate_retrieval(sample_dataset, sample_retrieval_results, config, k_values=[1, 2, 3]) + + # Verify metrics structure + assert isinstance(metrics, EvaluationMetrics) + assert 1 in metrics.precision_at_k + assert 2 in metrics.precision_at_k + assert 3 in metrics.precision_at_k + + # Verify metrics are reasonable (between 0 and 1) + assert 0 <= metrics.mean_average_precision <= 1 + assert 0 <= metrics.mean_reciprocal_rank <= 1 + + for k in [1, 2, 3]: + assert 0 <= metrics.precision_at_k[k] <= 1 + assert 0 <= metrics.recall_at_k[k] <= 1 + assert 0 <= metrics.f1_at_k[k] <= 1 + assert 0 <= metrics.ndcg_at_k[k] <= 1 + + def test_evaluate_retrieval_missing_results(self, engine, sample_dataset): + """Test evaluation with missing retrieval results""" + incomplete_results = { + "q1": [{"id": "chunk_1", "score": 0.9}] + # Missing q2 results + } + + config = RetrievalConfig(top_k=5) + + # Should not crash, should handle missing results gracefully + metrics = engine.evaluate_retrieval(sample_dataset, incomplete_results, config) + + # Should still return valid metrics structure + assert isinstance(metrics, EvaluationMetrics) + + def test_compare_evaluations(self, engine): + """Test evaluation comparison""" + eval1 = EvaluationMetrics( + precision_at_k={5: 0.6}, + recall_at_k={5: 0.5}, + f1_at_k={5: 0.55}, + mean_reciprocal_rank=0.7, + mean_average_precision=0.6, + ndcg_at_k={5: 0.65}, + ) + + eval2 = EvaluationMetrics( + precision_at_k={5: 0.8}, + recall_at_k={5: 0.7}, + f1_at_k={5: 0.75}, + mean_reciprocal_rank=0.9, + mean_average_precision=0.8, + ndcg_at_k={5: 0.85}, + ) + + comparison = engine.compare_evaluations(eval1, eval2) + + # Check improvements (with floating point tolerance) + assert abs(comparison["map_improvement"] - 0.2) < 1e-10 + assert abs(comparison["mrr_improvement"] - 0.2) < 1e-10 + assert abs(comparison["precision_at_5_improvement"] - 0.2) < 1e-10 + assert abs(comparison["recall_at_5_improvement"] - 0.2) < 1e-10 + assert abs(comparison["f1_at_5_improvement"] - 0.2) < 1e-10 + assert abs(comparison["ndcg_at_5_improvement"] - 0.2) < 1e-10 + + def test_empty_evaluation(self, engine): + """Test evaluation with empty dataset""" + empty_dataset = BenchmarkDatasetContent( + dataset_id="empty", + name="Empty Dataset", + description="Empty", + version="1.0", + question_set="tcfd", + created_at="2024-01-01", + questions=[], + ) + + metrics = engine.evaluate_retrieval(empty_dataset, {}, RetrievalConfig()) + + # Should return empty metrics + assert isinstance(metrics, EvaluationMetrics) + assert metrics.mean_average_precision == 0.0 + assert metrics.mean_reciprocal_rank == 0.0 diff --git a/tests/test_benchmark_store.py b/tests/test_benchmark_store.py new file mode 100644 index 00000000..8117e6e0 --- /dev/null +++ b/tests/test_benchmark_store.py @@ -0,0 +1,310 @@ +import tempfile +from pathlib import Path + +import pytest + +from report_analyst.core.storage.benchmark_store import BenchmarkStore +from report_analyst.models.benchmark import ( + BenchmarkDatasetContent, + BenchmarkEvaluation, + BenchmarkQuestion, + EvaluationMetrics, + GroundTruthChunk, + HumanAnnotation, + RetrievalConfig, +) + + +class TestBenchmarkStore: + """Test suite for benchmark data storage""" + + @pytest.fixture + def temp_db(self): + """Create a temporary database for testing. + + The BenchmarkStore class is responsible for initializing the schema, + so the fixture only needs to create a temporary file and let + BenchmarkStore.init_db() handle table creation. + """ + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp_db: + db_path = tmp_db.name + + # Initialize database schema via BenchmarkStore + BenchmarkStore(db_path) + + try: + yield db_path + finally: + try: + Path(db_path).unlink() + except OSError: + # Temp DB may already be gone if setup failed mid-fixture. + pass + + @pytest.fixture + def store(self, temp_db): + return BenchmarkStore(temp_db) + + @pytest.fixture + def sample_dataset(self): + """Create a sample dataset for testing""" + chunks = [ + GroundTruthChunk( + chunk_id="chunk_1", + relevance_score=1.0, + is_evidence=True, + evidence_order=1, + annotation_notes="High relevance chunk", + ), + GroundTruthChunk( + chunk_id="chunk_2", + relevance_score=0.8, + is_evidence=True, + evidence_order=2, + annotation_notes="Medium relevance chunk", + ), + ] + + questions = [ + BenchmarkQuestion( + question_id="tcfd_1", + question_text="What are the climate risks?", + ground_truth_chunks=chunks, + ) + ] + + return BenchmarkDatasetContent( + dataset_id="test_dataset_v1", + name="Test TCFD Dataset", + description="Test dataset for TCFD questions", + version="1.0", + question_set="tcfd", + created_at="2024-01-15", + questions=questions, + ) + + def test_save_and_get_dataset(self, store, sample_dataset): + """Test saving and retrieving a dataset""" + # Save dataset + dataset_id = store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + assert dataset_id is not None + + # Retrieve dataset + retrieved = store.get_dataset(sample_dataset.dataset_id) + assert retrieved is not None + assert retrieved.dataset_id == sample_dataset.dataset_id + assert retrieved.name == sample_dataset.name + assert retrieved.description == sample_dataset.description + assert retrieved.version == sample_dataset.version + assert retrieved.question_set == sample_dataset.question_set + assert retrieved.file_path == "/path/to/dataset.yaml" + + def test_get_nonexistent_dataset(self, store): + """Test retrieving a dataset that doesn't exist""" + result = store.get_dataset("nonexistent_dataset") + assert result is None + + def test_list_datasets(self, store, sample_dataset): + """Test listing datasets""" + # Initially empty + datasets = store.list_datasets() + assert len(datasets) == 0 + + # Save a dataset + store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + + # Should now have one dataset + datasets = store.list_datasets() + assert len(datasets) == 1 + assert datasets[0].dataset_id == sample_dataset.dataset_id + + def test_get_ground_truth(self, store, sample_dataset): + """Test retrieving ground truth data""" + # Save dataset + store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + + # Get ground truth + ground_truth = store.get_ground_truth(sample_dataset.dataset_id) + + assert "tcfd_1" in ground_truth + question_gt = ground_truth["tcfd_1"] + assert "chunk_1" in question_gt + assert "chunk_2" in question_gt + assert question_gt["chunk_1"] == 1.0 + assert question_gt["chunk_2"] == 0.8 + + def test_save_and_get_evaluation(self, store, sample_dataset): + """Test saving and retrieving evaluations""" + # First save a dataset + store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + + # Create evaluation + config = RetrievalConfig( + chunk_size=1000, + chunk_overlap=200, + top_k=5, + use_llm_scoring=True, + llm_model="gpt-4o-mini", + ) + + metrics = EvaluationMetrics( + precision_at_k={1: 0.8, 5: 0.6}, + recall_at_k={1: 0.2, 5: 0.5}, + f1_at_k={1: 0.32, 5: 0.55}, + mean_reciprocal_rank=0.75, + mean_average_precision=0.65, + ndcg_at_k={1: 0.8, 5: 0.68}, + ) + + evaluation = BenchmarkEvaluation( + dataset_id=sample_dataset.dataset_id, + evaluation_name="test_evaluation", + config_hash="test_hash", + retrieval_config=config, + evaluation_metrics=metrics, + ) + + # Save evaluation + eval_id = store.save_evaluation(evaluation) + assert eval_id is not None + + # Retrieve evaluation + retrieved = store.get_evaluation(eval_id) + assert retrieved is not None + assert retrieved.dataset_id == sample_dataset.dataset_id + assert retrieved.evaluation_name == "test_evaluation" + assert retrieved.retrieval_config.chunk_size == 1000 + assert retrieved.evaluation_metrics.mean_average_precision == 0.65 + + def test_list_evaluations(self, store, sample_dataset): + """Test listing evaluations""" + # Save dataset + store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + + # Initially empty + evaluations = store.list_evaluations() + assert len(evaluations) == 0 + + # Save evaluation + evaluation = BenchmarkEvaluation( + dataset_id=sample_dataset.dataset_id, + evaluation_name="test_eval", + config_hash="hash", + retrieval_config=RetrievalConfig(), + evaluation_metrics=EvaluationMetrics(), + ) + store.save_evaluation(evaluation) + + # Should now have one evaluation + evaluations = store.list_evaluations() + assert len(evaluations) == 1 + assert evaluations[0].evaluation_name == "test_eval" + + # Test filtering by dataset + filtered = store.list_evaluations(dataset_id=sample_dataset.dataset_id) + assert len(filtered) == 1 + + filtered_empty = store.list_evaluations(dataset_id="nonexistent") + assert len(filtered_empty) == 0 + + def test_save_and_get_annotations(self, store, sample_dataset): + """Test saving and retrieving human annotations""" + # Save dataset and evaluation first + store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + + evaluation = BenchmarkEvaluation( + dataset_id=sample_dataset.dataset_id, + evaluation_name="test_eval", + config_hash="hash", + retrieval_config=RetrievalConfig(), + evaluation_metrics=EvaluationMetrics(), + ) + eval_id = store.save_evaluation(evaluation) + + # Create annotation + annotation = HumanAnnotation( + evaluation_id=eval_id, + question_id="tcfd_1", + chunk_id="chunk_1", + human_relevance_score=0.9, + human_is_evidence=True, + human_evidence_order=1, + annotation_notes="Human annotation notes", + annotator_id="annotator_1", + ) + + # Save annotation + annotation_id = store.save_annotation(annotation) + assert annotation_id is not None + + # Retrieve annotations + annotations = store.get_annotations(eval_id) + assert len(annotations) == 1 + + retrieved = annotations[0] + assert retrieved.evaluation_id == eval_id + assert retrieved.question_id == "tcfd_1" + assert retrieved.chunk_id == "chunk_1" + assert retrieved.human_relevance_score == 0.9 + assert retrieved.human_is_evidence is True + assert retrieved.annotator_id == "annotator_1" + + def test_delete_dataset(self, store, sample_dataset): + """Test deleting a dataset and all related data""" + # Save dataset + store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + + # Save evaluation + evaluation = BenchmarkEvaluation( + dataset_id=sample_dataset.dataset_id, + evaluation_name="test_eval", + config_hash="hash", + retrieval_config=RetrievalConfig(), + evaluation_metrics=EvaluationMetrics(), + ) + eval_id = store.save_evaluation(evaluation) + + # Save annotation + annotation = HumanAnnotation( + evaluation_id=eval_id, + question_id="tcfd_1", + chunk_id="chunk_1", + human_relevance_score=0.9, + human_is_evidence=True, + annotator_id="annotator_1", + ) + store.save_annotation(annotation) + + # Verify data exists + assert store.get_dataset(sample_dataset.dataset_id) is not None + assert len(store.list_evaluations(sample_dataset.dataset_id)) == 1 + assert len(store.get_annotations(eval_id)) == 1 + + # Delete dataset + deleted = store.delete_dataset(sample_dataset.dataset_id) + assert deleted is True + + # Verify all related data is deleted + assert store.get_dataset(sample_dataset.dataset_id) is None + assert len(store.list_evaluations(sample_dataset.dataset_id)) == 0 + # Note: annotations are also deleted via foreign key cascade + + def test_delete_nonexistent_dataset(self, store): + """Test deleting a dataset that doesn't exist""" + deleted = store.delete_dataset("nonexistent_dataset") + assert deleted is False + + def test_save_dataset_replace_existing(self, store, sample_dataset): + """Test that saving a dataset with same ID replaces the existing one""" + # Save dataset + store.save_dataset(sample_dataset, "/path/to/dataset.yaml") + + # Modify and save again + sample_dataset.name = "Updated Dataset Name" + store.save_dataset(sample_dataset, "/path/to/updated_dataset.yaml") + + # Should have updated the existing record + datasets = store.list_datasets() + assert len(datasets) == 1 + assert datasets[0].name == "Updated Dataset Name" + assert datasets[0].file_path == "/path/to/updated_dataset.yaml" diff --git a/tests/test_benchmarking_ui.py b/tests/test_benchmarking_ui.py new file mode 100644 index 00000000..21142ae2 --- /dev/null +++ b/tests/test_benchmarking_ui.py @@ -0,0 +1,612 @@ +"""Unit tests for BenchmarkingUI with Streamlit mocked.""" + +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from report_analyst.models.benchmark import ( + BenchmarkDataset, + BenchmarkEvaluation, + DatasetType, + EvaluationMetrics, + FlexibleDatasetRow, + RetrievalConfig, +) +from report_analyst.ui.benchmarking import BenchmarkingUI + + +class _SessionState(dict): + """Minimal stand-in for streamlit.session_state attribute + item access.""" + + def __getattr__(self, key): + try: + return self[key] + except KeyError as exc: + raise AttributeError(key) from exc + + def __setattr__(self, key, value): + self[key] = value + + def __delattr__(self, key): + del self[key] + + +@pytest.fixture +def cache_manager(tmp_path): + cm = MagicMock() + cm.db_path = str(tmp_path / "bench.db") + return cm + + +@pytest.fixture +def ui(cache_manager): + with patch("report_analyst.ui.benchmarking.BenchmarkStore") as store_cls: + store_cls.return_value = MagicMock() + return BenchmarkingUI(cache_manager) + + +@pytest.fixture +def mock_st(monkeypatch): + st = MagicMock() + st.session_state = _SessionState() + + def _columns(n): + cols = [] + for _ in range(n): + col = MagicMock() + col.__enter__ = MagicMock(return_value=col) + col.__exit__ = MagicMock(return_value=False) + cols.append(col) + return cols + + st.columns.side_effect = _columns + + def _tabs(labels): + tabs = [] + for _ in labels: + tab = MagicMock() + tab.__enter__ = MagicMock(return_value=tab) + tab.__exit__ = MagicMock(return_value=False) + tabs.append(tab) + return tabs + + st.tabs.side_effect = _tabs + expander = MagicMock() + expander.__enter__ = MagicMock(return_value=expander) + expander.__exit__ = MagicMock(return_value=False) + st.expander.return_value = expander + spinner = MagicMock() + spinner.__enter__ = MagicMock(return_value=spinner) + spinner.__exit__ = MagicMock(return_value=False) + st.spinner.return_value = spinner + monkeypatch.setattr("report_analyst.ui.benchmarking.st", st) + return st + + +def _sample_dataset(dataset_id="d1", n=2): + rows = [ + FlexibleDatasetRow(data={"query_id": "q1", "chunk_id": f"c{i}", "position": i, "score": 0.9 - i * 0.1}) + for i in range(1, n + 1) + ] + return BenchmarkDataset( + dataset_id=dataset_id, + name="Sample", + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + results=rows, + ) + + +def test_ui_init_wires_dependencies(ui, cache_manager): + assert ui.cache_manager is cache_manager + assert ui.evaluation_engine is not None + assert ui.dataset_loader is not None + + +def test_render_csv_metrics_table_and_charts(ui, mock_st): + metrics = EvaluationMetrics( + precision_at_k={1: 0.5, 5: 0.4}, + recall_at_k={1: 0.5, 5: 0.4}, + f1_at_k={1: 0.5, 5: 0.4}, + mean_reciprocal_rank=0.5, + mean_average_precision=0.4, + ndcg_at_k={1: 0.5, 5: 0.4}, + ) + ui._render_csv_metrics_table(metrics, [1, 5]) + with patch("report_analyst.ui.benchmarking.px") as px: + px.line.return_value = MagicMock() + px.bar.return_value = MagicMock() + ui._render_csv_metrics_charts(metrics, [1, 5], chart_key_prefix="t") + assert mock_st.dataframe.called or mock_st.table.called or mock_st.write.called + + +def test_run_csv_evaluation_from_datasets_success(ui, mock_st): + ref = _sample_dataset("ref") + bench = _sample_dataset("bench") + mock_st.session_state.uploaded_datasets = {"ref": ref, "bench": bench} + + with patch.object(ui.evaluation_engine, "compare_flexible_datasets") as compare: + compare.return_value = EvaluationMetrics(precision_at_k={1: 1.0}, mean_average_precision=1.0) + with patch.object(ui, "_render_csv_metrics_table"), patch.object(ui, "_render_csv_metrics_charts"): + ui._run_csv_evaluation_from_datasets("upload", "ref", "upload", "bench", [1, 5], "eval1") + + assert "csv_evaluations" in mock_st.session_state + assert mock_st.session_state["csv_evaluations"][0].evaluation_name == "eval1" + assert mock_st.success.called + + +def test_run_csv_evaluation_db_source_errors(ui, mock_st): + ui._run_csv_evaluation_from_datasets("db", "x", "upload", "y", [1], "e") + assert mock_st.error.called + + +def test_run_csv_evaluation_missing_upload(ui, mock_st): + mock_st.session_state.uploaded_datasets = {} + ui._run_csv_evaluation_from_datasets("upload", "missing", "upload", "missing", [1], "e") + assert mock_st.error.called + + +def test_render_confirmation_ui_confirm_and_cancel(ui, mock_st): + dataset = _sample_dataset() + mock_st.session_state.uploaded_datasets = {"tmp": dataset, "ground_truth_current": dataset} + mock_st.button.side_effect = [True, False] # confirm + ui._render_confirmation_ui(dataset, "tmp", "ground_truth", "f.csv") + assert "ground_truth_current" in mock_st.session_state.uploaded_datasets + assert mock_st.rerun.called + + mock_st.button.side_effect = [False, True] # cancel + mock_st.session_state.uploaded_datasets = {"tmp": dataset} + mock_st.rerun.reset_mock() + ui._render_confirmation_ui(dataset, "tmp", "ground_truth", "f.csv") + assert "tmp" not in mock_st.session_state.uploaded_datasets + + +def test_render_results_table_and_details(ui, mock_st): + metrics = EvaluationMetrics(precision_at_k={1: 0.5}, mean_average_precision=0.5) + evaluation = BenchmarkEvaluation( + dataset_id="d1", + evaluation_name="e1", + config_hash="abc", + retrieval_config=RetrievalConfig(top_k=5), + evaluation_metrics=metrics, + ) + ui._render_results_table([evaluation]) + ui._render_evaluation_details(evaluation) + assert mock_st.write.called or mock_st.dataframe.called or mock_st.metric.called + + +def test_render_metrics_charts(ui, mock_st): + metrics = EvaluationMetrics( + precision_at_k={1: 0.5, 5: 0.4}, + recall_at_k={1: 0.5, 5: 0.4}, + f1_at_k={1: 0.5, 5: 0.4}, + ndcg_at_k={1: 0.5, 5: 0.4}, + mean_reciprocal_rank=0.5, + mean_average_precision=0.4, + ) + evaluation = BenchmarkEvaluation( + dataset_id="d1", + evaluation_name="e1", + config_hash="abc", + retrieval_config=RetrievalConfig(top_k=5), + evaluation_metrics=metrics, + ) + with patch("report_analyst.ui.benchmarking.px") as px: + px.line.return_value = MagicMock() + px.bar.return_value = MagicMock() + ui._render_metrics_charts([evaluation]) + + +def test_show_and_delete_dataset(ui, mock_st): + ui.benchmark_store.get_dataset.return_value = MagicMock(name="n", dataset_id="d1", description="x", question_set="tcfd") + ui.benchmark_store.get_dataset_content.return_value = MagicMock(questions=[1, 2, 3]) + ui._show_dataset_details("d1") + ui.benchmark_store.delete_dataset.return_value = True + ui._delete_dataset("d1") + assert ui.benchmark_store.delete_dataset.called + + +def test_render_config_form(ui, mock_st): + mock_st.number_input.side_effect = [1000, 200, 5, 0.0] + mock_st.checkbox.return_value = False + mock_st.selectbox.return_value = "default" + mock_st.text_input.return_value = None + cfg = ui._render_config_form() + assert isinstance(cfg, RetrievalConfig) + assert cfg.top_k == 5 + + +def test_handle_dataset_upload_csv(ui, mock_st): + mock_st.session_state.uploaded_datasets = {} + uploaded = MagicMock() + uploaded.name = "data.csv" + uploaded.getvalue.return_value = b"query_id,chunk_id,position,score\nq1,c1,1,0.9\n" + + with patch( + "report_analyst.ui.benchmarking.load_flexible_dataset_from_csv", + return_value=_sample_dataset("up"), + ): + ui._handle_dataset_upload(uploaded, dataset_type="ground_truth") + + assert mock_st.session_state.get("uploaded_datasets") is not None or mock_st.success.called or mock_st.error.called + + +def test_render_annotation_interface_and_form(ui, mock_st): + ui.render_annotation_interface() + metrics = EvaluationMetrics() + evaluation = BenchmarkEvaluation( + dataset_id="d1", + evaluation_name="e1", + config_hash="abc", + retrieval_config=RetrievalConfig(), + evaluation_metrics=metrics, + ) + ui._render_annotation_form(evaluation) + + +def test_render_dataset_management_smoke(ui, mock_st): + mock_st.radio.return_value = "Ranking (retrieval)" + mock_st.file_uploader.return_value = None + mock_st.selectbox.return_value = None + ui.benchmark_store.list_datasets.return_value = [] + ui.render_dataset_management() + assert mock_st.subheader.called + + +def test_render_benchmarking_interface_smoke(ui, mock_st): + mock_st.session_state.uploaded_datasets = { + "ground_truth_current": _sample_dataset("gt"), + "benchmark_current": _sample_dataset("bm"), + } + mock_st.selectbox.side_effect = ["upload", "ground_truth_current", "upload", "benchmark_current"] + mock_st.text_input.side_effect = ["my eval", "1,5"] + mock_st.button.return_value = False + ui.render_benchmarking_interface() + assert mock_st.subheader.called or mock_st.write.called + + +def test_render_results_dashboard_smoke(ui, mock_st): + mock_st.session_state.csv_evaluations = [] + ui.benchmark_store.list_evaluations.return_value = [] + ui.render_results_dashboard() + + +def _csv_upload(name: str, content: str): + uploaded = MagicMock() + uploaded.name = name + uploaded.getvalue.return_value = content.encode("utf-8") + # file-like for pd.read_csv(gt_file) + from io import StringIO + + buf = StringIO(content) + uploaded.__iter__ = buf.__iter__ + # pandas read_csv can take file-like; attach read + uploaded.read = lambda *a, **k: content.encode("utf-8") + return uploaded + + +def test_handle_classification_upload(ui, mock_st): + df = pd.DataFrame({"relevance": [0, 1, 2], "score_a": [0.1, 0.8, 0.9]}) + uploaded = MagicMock(name="cls.csv") + uploaded.name = "cls.csv" + ui._handle_classification_upload(df, uploaded, "classification") + assert "classification_current" in mock_st.session_state.uploaded_datasets + assert mock_st.success.called + + +def test_handle_classification_upload_empty(ui, mock_st): + ui._handle_classification_upload(pd.DataFrame(), MagicMock(name="e.csv"), "classification") + assert mock_st.error.called + + +def test_handle_dataset_upload_csv_confirm(ui, mock_st): + mock_st.session_state.uploaded_datasets = {} + mock_st.button.side_effect = [True, False] + uploaded = MagicMock() + uploaded.name = "ok.csv" + uploaded.getvalue.return_value = b"query_id,chunk_id,position,score\nq1,c1,1,0.9\n" + with patch( + "report_analyst.ui.benchmarking.load_flexible_dataset_from_csv", + return_value=_sample_dataset("up"), + ): + ui._handle_dataset_upload(uploaded, "ground_truth") + assert "ground_truth_current" in mock_st.session_state.uploaded_datasets + + +def test_handle_dataset_upload_already_aligned(ui, mock_st): + ds = _sample_dataset("aligned") + mock_st.session_state["aligned_ground_truth_ok.csv"] = True + mock_st.session_state.uploaded_datasets = {"ground_truth_current": ds} + uploaded = MagicMock() + uploaded.name = "ok.csv" + uploaded.getvalue.return_value = b"query_id,chunk_id,position,score\nq1,c1,1,0.9\n" + ui._handle_dataset_upload(uploaded, "ground_truth") + assert mock_st.success.called + + +def test_render_classification_calibration_panel(ui, mock_st): + rows = [ + FlexibleDatasetRow(data={"relevance": 1, "pred": 0.8, "query_id": "q1", "chunk_id": "c1"}), + FlexibleDatasetRow(data={"relevance": 0, "pred": 0.2, "query_id": "q1", "chunk_id": "c2"}), + ] + ds = BenchmarkDataset( + dataset_id="c", + name="cls", + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + results=rows, + column_mapping={"classification_label_col": "relevance", "classification_prediction_cols": "pred"}, + ) + mock_st.session_state.uploaded_datasets = {"classification_current": ds} + + def _select(*args, **kwargs): + options = kwargs.get("options") or [] + index = kwargs.get("index", 0) + return options[index] if options else None + + mock_st.selectbox.side_effect = _select + mock_st.multiselect.return_value = ["pred"] + mock_st.slider.return_value = 10 + mock_st.button.return_value = True + + metrics_df = pd.DataFrame({"model": ["pred"], "ece": [0.1]}) + with patch("report_analyst.ui.benchmarking.compute_calibration_metrics", return_value=metrics_df), patch( + "report_analyst.ui.benchmarking.compute_classification_report", + return_value={"0": {"precision": 1.0}, "accuracy": 1.0}, + ): + ui._render_classification_calibration_panel("t_") + + assert mock_st.session_state.get("csv_classification_evaluations") + + +def test_render_classification_model_comparison(ui, mock_st): + mock_st.session_state.csv_classification_evaluations = [ + { + "evaluation_name": "run1", + "created_at": pd.Timestamp.now(), + "metrics_df": pd.DataFrame({"m": [1]}), + } + ] + mock_st.selectbox.side_effect = lambda *a, **k: 0 + ui._render_classification_model_comparison() + assert mock_st.dataframe.called + + +def test_render_error_analysis_export(ui, mock_st): + ref = _sample_dataset("ref") + bench = _sample_dataset("bench") + mock_st.session_state.uploaded_datasets = {"ref": ref, "bench": bench} + mock_st.session_state.csv_evaluations = [ + type( + "E", + (), + { + "retrieval_config": RetrievalConfig(top_k=2), + "ref_key": "ref", + "bench_key": "bench", + "dataset_id": "ref|||bench", + }, + )() + ] + mock_st.button.return_value = True + with patch( + "report_analyst.ui.benchmarking.build_error_analysis_dataframe_from_flexible", + return_value=pd.DataFrame({"a": [1]}), + ): + ui._render_error_analysis_export() + assert mock_st.download_button.called + + +def test_run_evaluation_placeholder(ui, mock_st): + ui.benchmark_store.save_evaluation.return_value = 42 + ui._run_evaluation("d1", "e1", RetrievalConfig()) + assert mock_st.success.called + + +def test_flexible_gt_wizard_aligns(ui, mock_st): + content = "document,question,context,relevance\nR,Q,chunk text,1\n" + uploaded = MagicMock() + uploaded.name = "gt.csv" + + def _select(*args, **kwargs): + options = kwargs.get("options") or [] + index = kwargs.get("index", 0) + return options[index] if options else None + + mock_st.file_uploader.return_value = uploaded + mock_st.selectbox.side_effect = _select + mock_st.multiselect.return_value = ["relevance"] + mock_st.button.return_value = True + + with patch("report_analyst.ui.benchmarking.pd.read_csv", return_value=pd.read_csv(__import__("io").StringIO(content))): + with patch( + "report_analyst.ui.benchmarking.align_ground_truth_flexible", + return_value=pd.DataFrame( + { + "query_id": ["R|||Q"], + "chunk_id": ["c1"], + "position": [1], + "score": [1.0], + } + ), + ): + ui._render_flexible_gt_wizard() + + assert "ground_truth_flexible_current" in mock_st.session_state.uploaded_datasets + + +def test_flexible_bm_wizard_no_file_returns(ui, mock_st): + mock_st.file_uploader.return_value = None + ui._render_flexible_bm_wizard() + + +def test_render_dataset_management_classification_mode(ui, mock_st): + mock_st.radio.return_value = "Classification" + mock_st.file_uploader.return_value = None + ui.benchmark_store.list_datasets.return_value = [] + with patch.object(ui, "_render_flexible_bm_wizard") as wiz: + ui.render_dataset_management() + assert wiz.called + + +def test_annotation_with_evaluations(ui, mock_st): + metrics = EvaluationMetrics() + evaluation = BenchmarkEvaluation( + dataset_id="d1", + evaluation_name="e1", + config_hash="abc", + retrieval_config=RetrievalConfig(), + evaluation_metrics=metrics, + ) + ui.benchmark_store.list_evaluations.return_value = [evaluation] + mock_st.selectbox.return_value = evaluation + ui.render_annotation_interface() + + +def test_flexible_bm_wizard_aligns_ranking_mode(ui, mock_st): + content = "report,question,paragraph,score\nR,Q,para text,0.9\n" + uploaded = MagicMock() + uploaded.name = "bm.csv" + + def _select(*args, **kwargs): + options = kwargs.get("options") or [] + index = kwargs.get("index", 0) + return options[index] if options else None + + mock_st.session_state.evaluation_mode = "Ranking (retrieval)" + mock_st.file_uploader.return_value = uploaded + mock_st.selectbox.side_effect = _select + mock_st.multiselect.return_value = ["score"] + mock_st.button.return_value = True + + aligned = pd.DataFrame( + { + "query_id": ["R|||Q"], + "chunk_id": ["c1"], + "position": [1], + "score": [0.9], + "paragraph": ["para text"], + } + ) + with patch( + "report_analyst.ui.benchmarking.pd.read_csv", + return_value=pd.read_csv(__import__("io").StringIO(content)), + ), patch( + "report_analyst.ui.benchmarking.align_benchmark_flexible", + return_value=aligned, + ): + ui._render_flexible_bm_wizard() + + assert any("benchmark" in k for k in mock_st.session_state.uploaded_datasets) + + +def test_flexible_bm_wizard_classification_mode(ui, mock_st): + content = "document,question,paragraph,relevance,pred\nR,Q,p,1,0.8\n" + uploaded = MagicMock() + uploaded.name = "bm.csv" + + def _select(*args, **kwargs): + options = kwargs.get("options") or [] + index = kwargs.get("index", 0) + return options[index] if options else None + + mock_st.session_state.evaluation_mode = "Classification" + mock_st.file_uploader.return_value = uploaded + mock_st.selectbox.side_effect = _select + mock_st.multiselect.return_value = ["pred"] + mock_st.button.return_value = True + + aligned = pd.DataFrame( + { + "query_id": ["R|||Q"], + "chunk_id": ["c1"], + "position": [1], + "score": [0.8], + "relevance": [1], + "pred": [0.8], + } + ) + with patch( + "report_analyst.ui.benchmarking.pd.read_csv", + return_value=pd.read_csv(__import__("io").StringIO(content)), + ), patch( + "report_analyst.ui.benchmarking.align_benchmark_flexible", + return_value=aligned, + ): + ui._render_flexible_bm_wizard() + + assert mock_st.success.called + + +def test_handle_yaml_dataset_upload(ui, mock_st): + mock_st.session_state.uploaded_datasets = {} + mock_st.button.return_value = True + uploaded = MagicMock() + uploaded.name = "ds.yaml" + uploaded.getvalue.return_value = b"dataset: x\n" + fake_ds = MagicMock() + fake_ds.name = "Y" + fake_ds.description = "d" + fake_ds.questions = [1, 2] + ui.dataset_loader = MagicMock() + ui.dataset_loader.load_dataset.return_value = fake_ds + ui.dataset_loader.validate_dataset_consistency.return_value = [] + ui._handle_dataset_upload(uploaded, "ground_truth") + assert ui.benchmark_store.save_dataset.called + + +def test_render_benchmarking_interface_runs_eval(ui, mock_st): + mock_st.session_state.uploaded_datasets = { + "ground_truth_current": _sample_dataset("gt"), + "benchmark_current": _sample_dataset("bm"), + } + + def _select(*args, **kwargs): + options = kwargs.get("options") or [] + index = kwargs.get("index", 0) + return options[index] if options else None + + mock_st.radio.return_value = "Ranking (retrieval)" + mock_st.selectbox.side_effect = _select + mock_st.text_input.side_effect = ["eval-run", "1,5"] + mock_st.number_input.return_value = 10 + mock_st.button.return_value = True + with patch.object(ui, "_run_csv_evaluation_from_datasets") as run: + ui.render_benchmarking_interface() + assert run.called + + +def test_render_results_dashboard_with_evals(ui, mock_st): + metrics = EvaluationMetrics( + precision_at_k={1: 0.5, 5: 0.4}, + recall_at_k={1: 0.5, 5: 0.4}, + f1_at_k={1: 0.5, 5: 0.4}, + ndcg_at_k={1: 0.5, 5: 0.4}, + mean_reciprocal_rank=0.5, + mean_average_precision=0.4, + ) + evaluation = BenchmarkEvaluation( + dataset_id="d1", + evaluation_name="e1", + config_hash="abc", + retrieval_config=RetrievalConfig(top_k=5), + evaluation_metrics=metrics, + ) + ui.benchmark_store.list_evaluations.return_value = [evaluation] + mock_st.session_state.csv_evaluations = [] + mock_st.multiselect.return_value = ["d1"] + mock_st.radio.return_value = "Both" + + def _select(*args, **kwargs): + options = kwargs.get("options") or [] + return options[0] if options else None + + mock_st.selectbox.side_effect = _select + with patch.object(ui, "_render_results_table"), patch.object(ui, "_render_metrics_charts"), patch.object( + ui, "_render_error_analysis_export" + ), patch.object(ui, "_render_classification_model_comparison"), patch.object( + ui, "_render_classification_calibration_panel" + ), patch.object( + ui, "_render_evaluation_details" + ) as details: + ui.render_results_dashboard() + assert details.called diff --git a/tests/test_cache_manager.py b/tests/test_cache_manager.py index 46d6fc6a..35dd9918 100644 --- a/tests/test_cache_manager.py +++ b/tests/test_cache_manager.py @@ -39,7 +39,7 @@ def test_init_db(temp_db): with sqlite3.connect(temp_db.db_path) as conn: cursor = conn.execute( """ - SELECT name FROM sqlite_master + SELECT name FROM sqlite_master WHERE type='table' AND (name='analysis_cache' OR name='document_chunks') """ ) @@ -171,8 +171,8 @@ def test_multiple_questions(temp_db): def test_error_handling(temp_db): """Test error handling""" - # Test invalid JSON - with pytest.raises(Exception): + # Test invalid JSON / non-dict result (AttributeError on .get / json.dumps TypeError) + with pytest.raises(Exception): # noqa: B017 temp_db.save_analysis( "test.pdf", "tcfd_1", @@ -181,7 +181,7 @@ def test_error_handling(temp_db): ) # Test missing required config params - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 temp_db.get_analysis("test.pdf", {"chunk_size": 500}) # Missing required params @@ -211,7 +211,6 @@ def test_cache_status(temp_db): def test_get_chunks_without_embeddings(temp_db): """Test get_chunks_without_embeddings method""" import sqlite3 - from datetime import datetime import numpy as np @@ -288,7 +287,6 @@ def test_get_chunks_without_embeddings(temp_db): def test_has_chunk_scoring(temp_db): """Test has_chunk_scoring method""" import sqlite3 - from datetime import datetime import numpy as np @@ -347,3 +345,109 @@ def test_has_chunk_scoring(temp_db): # Now should be True assert temp_db.has_chunk_scoring(file_path, config) is True + + +@pytest.mark.asyncio +async def test_vector_store_reloads_on_chunk_size_change(temp_db): + """Test that vector store reloads when chunk_size or chunk_overlap changes""" + import numpy as np + + file_path = "test_doc.pdf" + + # Use consistent embedding dimension for all chunks and query + embedding_dim = 384 # Common embedding dimension (e.g., sentence-transformers) + + # Create test chunks with chunk_size=500 + chunks_500 = [ + { + "text": "This is a test chunk for chunk size 500. " * 20, # Make it long enough + "embedding": np.random.rand(embedding_dim).astype(np.float32), + "metadata": {"page": 1, "chunk_size": 500, "chunk_overlap": 20}, + }, + { + "text": "Another test chunk for chunk size 500. " * 20, + "embedding": np.random.rand(embedding_dim).astype(np.float32), + "metadata": {"page": 2, "chunk_size": 500, "chunk_overlap": 20}, + }, + ] + + # Create test chunks with chunk_size=1000 (different chunking) + chunks_1000 = [ + { + "text": "This is a test chunk for chunk size 1000. " * 40, # Different content + "embedding": np.random.rand(embedding_dim).astype(np.float32), + "metadata": {"page": 1, "chunk_size": 1000, "chunk_overlap": 20}, + }, + { + "text": "Another test chunk for chunk size 1000. " * 40, + "embedding": np.random.rand(embedding_dim).astype(np.float32), + "metadata": {"page": 2, "chunk_size": 1000, "chunk_overlap": 20}, + }, + ] + + # Save chunks with chunk_size=500 + temp_db.save_vectors(file_path, chunks_500) + + # Get similar chunks with chunk_size=500 + query_embedding = np.random.rand(embedding_dim).astype(np.float32) + similar_chunks_500 = await temp_db.get_similar_chunks( + query_embedding=query_embedding, + file_path=file_path, + top_k=2, + chunk_size=500, + chunk_overlap=20, + ) + + # Verify we got chunks with chunk_size=500 + assert len(similar_chunks_500) > 0 + assert temp_db.current_chunk_size == 500 + assert temp_db.current_chunk_overlap == 20 + + # Save chunks with chunk_size=1000 + temp_db.save_vectors(file_path, chunks_1000) + + # Get similar chunks with chunk_size=1000 + # This should trigger a reload of the vector store + similar_chunks_1000 = await temp_db.get_similar_chunks( + query_embedding=query_embedding, + file_path=file_path, + top_k=2, + chunk_size=1000, + chunk_overlap=20, + ) + + # Verify the vector store was reloaded with new chunk_size + assert temp_db.current_chunk_size == 1000 + assert temp_db.current_chunk_overlap == 20 + + # Verify we got different chunks (or at least that the vector store was reloaded) + # The chunks should be different because we saved different chunks for chunk_size=1000 + assert len(similar_chunks_1000) > 0 + + # Test chunk_overlap change + chunks_diff_overlap = [ + { + "text": "This is a test chunk with different overlap. " * 20, + "embedding": np.random.rand(embedding_dim).astype(np.float32), + "metadata": { + "page": 1, + "chunk_size": 1000, + "chunk_overlap": 50, + }, # Different overlap + }, + ] + + temp_db.save_vectors(file_path, chunks_diff_overlap) + + # Get similar chunks with different chunk_overlap + await temp_db.get_similar_chunks( + query_embedding=query_embedding, + file_path=file_path, + top_k=2, + chunk_size=1000, + chunk_overlap=50, # Different overlap + ) + + # Verify the vector store was reloaded with new chunk_overlap + assert temp_db.current_chunk_overlap == 50 + assert temp_db.current_chunk_size == 1000 diff --git a/tests/test_classification_calibration.py b/tests/test_classification_calibration.py new file mode 100644 index 00000000..fbe4c8dd --- /dev/null +++ b/tests/test_classification_calibration.py @@ -0,0 +1,105 @@ +"""Unit tests for classification and calibration benchmark metrics.""" + +import numpy as np +import pandas as pd +import pytest + +from report_analyst.core.benchmark.classification_calibration import ( + compute_calibration_metrics, + compute_classification_report, + expected_calibration_error, + minmax_normalize, +) + + +class TestMinmaxNormalize: + """Tests for minmax_normalize.""" + + def test_returns_half_when_all_values_identical(self): + series = pd.Series([3, 3, 3]) + normalized = minmax_normalize(series) + np.testing.assert_array_equal(normalized, np.array([0.5, 0.5, 0.5])) + + def test_scales_values_to_unit_interval(self): + series = pd.Series([0, 5, 10]) + normalized = minmax_normalize(series) + np.testing.assert_allclose(normalized, np.array([0.0, 0.5, 1.0])) + + +class TestExpectedCalibrationError: + """Tests for expected_calibration_error.""" + + def test_returns_zero_for_empty_input(self): + assert expected_calibration_error(np.array([]), np.array([])) == 0.0 + + def test_perfect_calibration_has_low_error(self): + y_true = np.array([0, 0, 1, 1]) + y_prob = np.array([0.1, 0.2, 0.8, 0.9]) + ece = expected_calibration_error(y_true, y_prob, n_bins=2) + assert ece < 0.2 + + +class TestComputeCalibrationMetrics: + """Tests for compute_calibration_metrics.""" + + @pytest.fixture + def labeled_df(self): + return pd.DataFrame( + { + "ground_truth": [0, 1, 2, 0, 2], + "model_a": [0, 1, 2, 0, 2], + "model_b": [0, 0, 1, 0, 1], + } + ) + + def test_returns_one_row_per_prediction_column(self, labeled_df): + metrics = compute_calibration_metrics( + labeled_df, + ground_truth_col="ground_truth", + score_cols=["model_a", "model_b"], + n_bins=10, + ) + + assert len(metrics) == 2 + assert set(metrics["prediction"]) == {"model_a", "model_b"} + assert metrics.loc[metrics["prediction"] == "model_a", "f1_1"].iloc[0] == 1.0 + + def test_returns_nan_metrics_when_no_overlap(self): + df = pd.DataFrame({"ground_truth": [np.nan], "model_a": [1]}) + metrics = compute_calibration_metrics( + df, + ground_truth_col="ground_truth", + score_cols=["model_a"], + ) + + assert metrics.iloc[0]["ece"] != metrics.iloc[0]["ece"] # NaN + + +class TestComputeClassificationReport: + """Tests for compute_classification_report.""" + + def test_builds_three_class_report(self): + df = pd.DataFrame( + { + "relevance": [0, 1, 2, 0, 2], + "pred": [0, 1, 2, 1, 2], + } + ) + + report = compute_classification_report( + df, + ground_truth_col="relevance", + score_col="pred", + ) + + assert report["accuracy"] == pytest.approx(0.8) + assert "irrelevant (0)" in report + + def test_raises_when_columns_missing(self): + df = pd.DataFrame({"relevance": [0, 1]}) + with pytest.raises(ValueError, match="Columns 'relevance' and/or 'pred'"): + compute_classification_report( + df, + ground_truth_col="relevance", + score_col="pred", + ) diff --git a/tests/test_climretrieve_integration.py b/tests/test_climretrieve_integration.py new file mode 100644 index 00000000..69742145 --- /dev/null +++ b/tests/test_climretrieve_integration.py @@ -0,0 +1,224 @@ +""" +Integration test for ClimRetrieve benchmark datasets. + +This test downloads datasets from GitHub and runs the benchmark evaluation. +""" + +import logging +import sys +from pathlib import Path +from typing import Optional + +import pytest +import requests + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.core.benchmark.retrieval_results_loader import ( + load_flexible_dataset_from_csv, +) + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def climretrieve_data_dir(tmp_path): + """Create temporary directory for ClimRetrieve datasets""" + data_dir = tmp_path / "climretrieve" + data_dir.mkdir() + return data_dir + + +def download_github_file(repo: str, file_path: str, output_path: Path) -> Path: + """Download a file from GitHub raw content""" + url = f"https://raw.githubusercontent.com/{repo}/main/{file_path}" + + try: + response = requests.get(url, timeout=30) + response.raise_for_status() + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(response.content) + + return output_path + except requests.RequestException as e: + pytest.skip(f"Could not download from GitHub: {e}") + return output_path # pragma: no cover — pytest.skip always raises + + +def find_csv_in_directory(repo: str, directory: str) -> Optional[str]: + """Find first CSV file in a GitHub directory""" + # Try GitHub API first + api_url = f"https://api.github.com/repos/{repo}/contents/{directory}" + try: + response = requests.get(api_url, timeout=30) + response.raise_for_status() + contents = response.json() + + if isinstance(contents, dict): + contents = [contents] + + csv_files = [item["name"] for item in contents if item.get("type") == "file" and item["name"].endswith(".csv")] + + if csv_files: + return csv_files[0] + except requests.RequestException as exc: + logger.debug("GitHub API directory listing failed: %s", exc) + + # Fallback: try common names + common_names = ["dataset.csv", "data.csv", "results.csv", "benchmark.csv"] + for name in common_names: + test_url = f"https://raw.githubusercontent.com/{repo}/main/{directory}/{name}" + try: + response = requests.head(test_url, timeout=10) + if response.status_code == 200: + return name + except requests.RequestException as exc: + logger.debug("GitHub HEAD check failed for %s: %s", test_url, exc) + continue + + return None + + +@pytest.mark.integration +def test_climretrieve_reference_dataset_download(climretrieve_data_dir): + """Test downloading ClimRetrieve reference dataset""" + repo = "tobischimanski/ClimRetrieve" + directory = "Expert-Annotated Relevant Sources Dataset" + + csv_file = find_csv_in_directory(repo, directory) + if csv_file is None: + pytest.skip("Could not find CSV file in reference dataset directory") + + file_path = f"{directory}/{csv_file}" + output_path = climretrieve_data_dir / f"reference_{csv_file}" + + downloaded = download_github_file(repo, file_path, output_path) + + assert downloaded.exists() + assert downloaded.stat().st_size > 0 + + +@pytest.mark.integration +def test_climretrieve_input_dataset_download(climretrieve_data_dir): + """Test downloading ClimRetrieve input dataset""" + repo = "tobischimanski/ClimRetrieve" + directory = "Report-Level Dataset" + + csv_file = find_csv_in_directory(repo, directory) + if csv_file is None: + pytest.skip("Could not find CSV file in input dataset directory") + + file_path = f"{directory}/{csv_file}" + output_path = climretrieve_data_dir / f"input_{csv_file}" + + downloaded = download_github_file(repo, file_path, output_path) + + assert downloaded.exists() + assert downloaded.stat().st_size > 0 + + +@pytest.mark.integration +def test_climretrieve_dataset_loading(climretrieve_data_dir): + """Test loading ClimRetrieve datasets""" + repo = "tobischimanski/ClimRetrieve" + + # Download reference dataset + ref_dir = "Expert-Annotated Relevant Sources Dataset" + ref_csv = find_csv_in_directory(repo, ref_dir) + if ref_csv is None: + pytest.skip("Could not find reference dataset") + + ref_path = climretrieve_data_dir / f"reference_{ref_csv}" + download_github_file(repo, f"{ref_dir}/{ref_csv}", ref_path) + + # Download input dataset + input_dir = "Report-Level Dataset" + input_csv = find_csv_in_directory(repo, input_dir) + if input_csv is None: + pytest.skip("Could not find input dataset") + + input_path = climretrieve_data_dir / f"input_{input_csv}" + download_github_file(repo, f"{input_dir}/{input_csv}", input_path) + + # Load datasets + reference = load_flexible_dataset_from_csv( + csv_path=str(ref_path), + dataset_id="climretrieve_reference", + dataset_name="ClimRetrieve Reference", + ) + + assert reference is not None + assert len(reference.results) > 0 + + input_dataset = load_flexible_dataset_from_csv( + csv_path=str(input_path), + dataset_id="climretrieve_input", + dataset_name="ClimRetrieve Input", + ) + + assert input_dataset is not None + assert len(input_dataset.results) > 0 + + +@pytest.mark.integration +def test_climretrieve_benchmark_evaluation(climretrieve_data_dir): + """Test running benchmark evaluation on ClimRetrieve datasets""" + repo = "tobischimanski/ClimRetrieve" + + # Download reference dataset + ref_dir = "Expert-Annotated Relevant Sources Dataset" + ref_csv = find_csv_in_directory(repo, ref_dir) + if ref_csv is None: + pytest.skip("Could not find reference dataset") + + ref_path = climretrieve_data_dir / f"reference_{ref_csv}" + download_github_file(repo, f"{ref_dir}/{ref_csv}", ref_path) + + # Download input dataset + input_dir = "Report-Level Dataset" + input_csv = find_csv_in_directory(repo, input_dir) + if input_csv is None: + pytest.skip("Could not find input dataset") + + input_path = climretrieve_data_dir / f"input_{input_csv}" + download_github_file(repo, f"{input_dir}/{input_csv}", input_path) + + # Load datasets + reference = load_flexible_dataset_from_csv( + csv_path=str(ref_path), + dataset_id="climretrieve_reference", + dataset_name="ClimRetrieve Reference", + ) + + input_dataset = load_flexible_dataset_from_csv( + csv_path=str(input_path), + dataset_id="climretrieve_input", + dataset_name="ClimRetrieve Input", + ) + + # Run evaluation + engine = EvaluationEngine() + metrics = engine.compare_flexible_datasets( + reference_dataset=reference, input_dataset=input_dataset, k_values=[1, 3, 5, 10] + ) + + # Assertions + assert metrics is not None + assert metrics.mean_average_precision >= 0.0 + assert metrics.mean_average_precision <= 1.0 + assert metrics.mean_reciprocal_rank >= 0.0 + assert metrics.mean_reciprocal_rank <= 1.0 + + # Check that we have metrics for at least one K value + assert len(metrics.precision_at_k) > 0 + assert len(metrics.recall_at_k) > 0 + + # Verify metrics are in valid range + for k, score in metrics.precision_at_k.items(): + assert 0.0 <= score <= 1.0, f"Precision@{k} out of range: {score}" + + for k, score in metrics.recall_at_k.items(): + assert 0.0 <= score <= 1.0, f"Recall@{k} out of range: {score}" diff --git a/tests/test_dataset_mapper.py b/tests/test_dataset_mapper.py new file mode 100644 index 00000000..58c62695 --- /dev/null +++ b/tests/test_dataset_mapper.py @@ -0,0 +1,82 @@ +import pandas as pd + +from report_analyst.core.benchmark.dataset_mapper import ( + DatasetMapperFactory, + DefaultDatasetMapper, + list_available_dataset_ids, +) + + +def test_list_available_dataset_ids_includes_climretrieve(): + dataset_ids = list_available_dataset_ids() + assert "climretrieve" in dataset_ids + + +def test_factory_returns_default_mapper_for_climretrieve(): + mapper = DatasetMapperFactory.get_mapper("climretrieve") + assert isinstance(mapper, DefaultDatasetMapper) + assert mapper.dataset_id == "climretrieve" + + +def test_default_mapper_aligns_ground_truth_climretrieve(): + # Minimal synthetic ground truth with expected columns for ClimRetrieve + df_raw = pd.DataFrame( + { + "document": ["Report A", "Report A"], + "question": ["What is X?", "What is X?"], + "context": ["before relevant after", "before relevant2 after2"], + "relevant": ["relevant", "relevant2"], + "page_number": [1, 2], + "Source Relevance Score": [2, 1], + } + ) + + mapper = DatasetMapperFactory.get_mapper("climretrieve") + df_aligned = mapper.align_ground_truth(df_raw) + + # Basic shape/columns checks + assert len(df_aligned) == len(df_raw) + for col in ["query_id", "chunk_id", "position", "score", "document", "question"]: + assert col in df_aligned.columns + + # Scores should reflect the source relevance scores + assert set(df_aligned["score"].tolist()) == {1.0, 2.0} + + # query_id should be consistent for rows with same (document, question) + assert df_aligned["query_id"].nunique() == 1 + + +def test_default_mapper_aligns_benchmark_climretrieve(): + df_raw = pd.DataFrame( + { + "report": ["Report A", "Report A"], + "question": ["What is X?", "What is X?"], + "paragraph": ["para1 text", "para2 text"], + "relevant_text": ["rel1", "rel2"], + "label": [2, 0], + } + ) + + mapper = DatasetMapperFactory.get_mapper("climretrieve") + df_aligned = mapper.align_benchmark(df_raw) + + # Basic shape/columns checks + assert len(df_aligned) == len(df_raw) + for col in [ + "query_id", + "chunk_id", + "relevant_part_id", + "position", + "report", + "question", + "paragraph", + "relevant_text", + "relevance_label", + ]: + assert col in df_aligned.columns + + # query_id should be consistent for rows with same (report, question) + assert df_aligned["query_id"].nunique() == 1 + + # Ensure relevant_part_id is derived from relevant_text (and thus differs from chunk_id) + assert not (df_aligned["chunk_id"] == df_aligned["relevant_part_id"]).all() diff --git a/tests/test_dataset_mapper_coverage.py b/tests/test_dataset_mapper_coverage.py new file mode 100644 index 00000000..32afb4c6 --- /dev/null +++ b/tests/test_dataset_mapper_coverage.py @@ -0,0 +1,77 @@ +"""Additional coverage for dataset mapper edge cases.""" + +import pandas as pd +import pytest + +from report_analyst.core.benchmark.dataset_mapper import ( + DatasetMapperFactory, + generate_chunk_id, + generate_query_id, + list_available_dataset_ids, + transform_benchmark_results, + transform_ground_truth, +) + + +def test_generate_query_id_strips_ends(): + assert generate_query_id(" Doc A ", " What? ") == "Doc A|||What?" + + +def test_generate_chunk_id_stable_and_empty(): + assert generate_chunk_id("hello") == generate_chunk_id("hello") + assert generate_chunk_id("") != "" + assert generate_chunk_id("x", prefix="p_").startswith("p_") + + +def test_transform_ground_truth_string_labels_and_missing_label(): + df = pd.DataFrame( + { + "Document": ["A", "A", "A"], + "Question": ["Q", "Q", "Q"], + "Relevant": ["r1", "r2", "r3"], + "Label": ["yes", "maybe", "no"], + } + ) + out = transform_ground_truth(df, relevance_label_col="Label") + assert set(out["score"].tolist()) == {2.0, 1.0, 0.0} + + df2 = pd.DataFrame({"document": ["A"], "question": ["Q"], "context": ["ctx only"]}) + out2 = transform_ground_truth(df2) + assert out2.iloc[0]["score"] == 1.0 + + +def test_transform_ground_truth_raises_without_required_columns(): + with pytest.raises(ValueError, match="document"): + transform_ground_truth(pd.DataFrame({"question": ["Q"], "relevant": ["r"]})) + with pytest.raises(ValueError, match="question"): + transform_ground_truth(pd.DataFrame({"document": ["D"], "relevant": ["r"]})) + with pytest.raises(ValueError, match="context or relevant"): + transform_ground_truth(pd.DataFrame({"document": ["D"], "question": ["Q"]})) + + +def test_transform_benchmark_results_number_col_and_no_relevant_text(): + df = pd.DataFrame( + { + "report": ["R"], + "question": ["Q"], + "paragraph": ["para text"], + "number": [0], + "label": ["high"], + } + ) + out = transform_benchmark_results(df) + assert out.iloc[0]["position"] == 1 + assert out.iloc[0]["relevant_part_id"] == out.iloc[0]["chunk_id"] + assert out.iloc[0]["score"] == 2.0 + + +def test_transform_benchmark_results_raises_without_paragraph(): + with pytest.raises(ValueError, match="paragraph"): + transform_benchmark_results(pd.DataFrame({"report": ["R"], "question": ["Q"]})) + + +def test_factory_unknown_dataset_falls_back_to_default(): + mapper = DatasetMapperFactory.get_mapper("unknown_dataset_xyz") + assert mapper.dataset_id == "unknown_dataset_xyz" + ids = list_available_dataset_ids() + assert isinstance(ids, list) diff --git a/tests/test_dataset_normalizer.py b/tests/test_dataset_normalizer.py new file mode 100644 index 00000000..4238cb37 --- /dev/null +++ b/tests/test_dataset_normalizer.py @@ -0,0 +1,106 @@ +"""Unit tests for benchmark dataset normalization helpers.""" + +import pandas as pd +import pytest + +from report_analyst.core.benchmark.dataset_mapper import ( + generate_chunk_id, + generate_query_id, +) +from report_analyst.core.benchmark.dataset_normalizer import ( + POSITION_MODE_ROW_ORDER, + POSITION_MODE_SORT_BY_SCORE, + make_chunk_id_from_text, + make_query_id_from_columns, + normalize_dataframe_for_benchmark, +) + + +class TestDatasetNormalizerHelpers: + """Tests for small ID/score helper functions.""" + + def test_make_query_id_from_columns_uses_document_when_present(self): + query_id = make_query_id_from_columns("doc_a", "What is scope 1?") + assert query_id == generate_query_id("doc_a", "What is scope 1?") + + def test_make_query_id_from_columns_falls_back_to_question_only(self): + query_id = make_query_id_from_columns(None, "What is scope 1?") + assert query_id == "What is scope 1?" + + def test_make_chunk_id_from_text_hashes_content(self): + chunk_id = make_chunk_id_from_text("Scope 1 emissions data") + assert chunk_id == generate_chunk_id("Scope 1 emissions data") + + +class TestNormalizeDataframeForBenchmark: + """Tests for normalize_dataframe_for_benchmark.""" + + @pytest.fixture + def raw_df(self): + return pd.DataFrame( + { + "description": ["q1", "q1", "q2"], + "paragraph": ["chunk a", "chunk b", "chunk c"], + "label": [2, "yes", "no"], + } + ) + + def test_row_order_assigns_positions_per_query(self, raw_df): + normalized = normalize_dataframe_for_benchmark( + raw_df, + query_column="description", + chunk_text_column="paragraph", + score_column="label", + position_mode=POSITION_MODE_ROW_ORDER, + ) + + assert list(normalized.columns) == [ + "query_id", + "chunk_id", + "position", + "score", + "paragraph", + "question", + ] + q1_rows = normalized[normalized["question"] == "q1"] + assert q1_rows["position"].tolist() == [1, 2] + assert q1_rows.iloc[0]["score"] == 2.0 + assert q1_rows.iloc[1]["score"] == 2.0 + + def test_sort_by_score_orders_rows_within_query(self, raw_df): + normalized = normalize_dataframe_for_benchmark( + raw_df, + query_column="description", + chunk_text_column="paragraph", + score_column="label", + position_mode=POSITION_MODE_SORT_BY_SCORE, + ) + + q1_rows = normalized[normalized["question"] == "q1"] + assert q1_rows.iloc[0]["paragraph"] == "chunk a" + assert q1_rows.iloc[0]["position"] == 1 + assert q1_rows.iloc[1]["position"] == 2 + + def test_document_column_builds_query_ids(self, raw_df): + raw_with_doc = raw_df.assign(report=["doc_a", "doc_a", "doc_b"]) + normalized = normalize_dataframe_for_benchmark( + raw_with_doc, + query_column="description", + chunk_text_column="paragraph", + score_column="label", + document_column="report", + position_mode=POSITION_MODE_ROW_ORDER, + ) + + assert normalized.iloc[0]["query_id"] == generate_query_id("doc_a", "q1") + assert normalized.iloc[2]["query_id"] == generate_query_id("doc_b", "q2") + assert normalized.iloc[0]["chunk_id"] == generate_chunk_id("chunk a") + + def test_raises_when_required_column_missing(self, raw_df): + with pytest.raises(ValueError, match="Query column 'missing'"): + normalize_dataframe_for_benchmark( + raw_df, + query_column="missing", + chunk_text_column="paragraph", + score_column="label", + ) diff --git a/tests/test_document_analyzer_construction.py b/tests/test_document_analyzer_construction.py new file mode 100644 index 00000000..e5c35e09 --- /dev/null +++ b/tests/test_document_analyzer_construction.py @@ -0,0 +1,16 @@ +"""Regression tests for DocumentAnalyzer construction API.""" + +from pathlib import Path + + +def test_streamlit_app_does_not_pass_cache_manager_to_document_analyzer(): + """DocumentAnalyzer.__init__ takes no kwargs; it builds CacheManager itself. + + Code Quality flagged DocumentAnalyzer(cache_manager=...) on the benchmarking + page. That call raises TypeError if executed — keep it out of source. + """ + source = Path("report_analyst/streamlit_app.py").read_text(encoding="utf-8") + assert "DocumentAnalyzer(cache_manager=" not in source, ( + "Do not pass cache_manager= to DocumentAnalyzer(); use DocumentAnalyzer() " + "or ReportAnalyzer() and read .cache_manager from the instance." + ) diff --git a/tests/test_error_analysis.py b/tests/test_error_analysis.py new file mode 100644 index 00000000..1c2eba1d --- /dev/null +++ b/tests/test_error_analysis.py @@ -0,0 +1,294 @@ +"""Unit tests for error-analysis dataframe builders.""" + +from types import SimpleNamespace + +from report_analyst.core.benchmark.error_analysis import ( + build_error_analysis_dataframe, + build_error_analysis_dataframe_from_flexible, +) +from report_analyst.models.benchmark import ( + BenchmarkDataset, + BenchmarkDatasetContent, + BenchmarkQuestion, + DatasetType, + FlexibleDatasetRow, + GroundTruthChunk, + RetrievalResultRow, +) + + +def _ground_truth_content() -> BenchmarkDatasetContent: + chunks = [ + GroundTruthChunk( + chunk_id="c1", + relevance_score=1.0, + is_evidence=True, + evidence_order=1, + annotation_notes="relevant text A", + text="relevant text A", + metadata={"document": "Report A"}, + ), + GroundTruthChunk( + chunk_id="c2", + relevance_score=0.0, + is_evidence=False, + annotation_notes="noise", + text="noise", + ), + ] + question = BenchmarkQuestion( + question_id="q1", + question_text="What is X?", + ground_truth_chunks=chunks, + ) + return BenchmarkDatasetContent( + dataset_id="gt", + name="GT", + description="test", + version="1.0", + question_set="tcfd", + created_at="2024-01-01", + questions=[question], + ) + + +def test_build_error_analysis_dataframe_marks_relevant_chunks(): + gt = _ground_truth_content() + retrieval = [ + RetrievalResultRow(query_id="q1", chunk_id="c1", position=1, score=0.9, chunk_text="hit"), + RetrievalResultRow(query_id="q1", chunk_id="c2", position=2, score=0.5, chunk_text="miss"), + RetrievalResultRow(query_id="q1", chunk_id="c3", position=3, score=0.1, chunk_text="extra"), + ] + + df = build_error_analysis_dataframe(gt, retrieval, top_k=2) + + assert len(df) == 2 + assert list(df["chunk_id"]) == ["c1", "c2"] + assert bool(df.iloc[0]["is_really_relevant"]) is True + assert bool(df.iloc[1]["is_really_relevant"]) is False + assert df.iloc[0]["question"] == "What is X?" + assert df.iloc[0]["report_name"] == "Report A" + assert df.iloc[0]["relevant_part_text"] == "relevant text A" + + +def test_build_error_analysis_dataframe_empty_when_no_results(): + gt = _ground_truth_content() + df = build_error_analysis_dataframe(gt, [], top_k=5) + assert df.empty + + +def test_build_error_analysis_dataframe_from_flexible_main_path(): + query_id = "Report A|||What is X?" + gt = BenchmarkDataset( + dataset_id="gt", + name="GT", + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + results=[ + FlexibleDatasetRow( + data={ + "query_id": query_id, + "chunk_id": "rel1", + "Relevant": "gold relevant text", + "document": "Report A", + "question": "What is X?", + "relevance_score": 2.0, + } + ) + ], + ) + bench = BenchmarkDataset( + dataset_id="bm", + name="BM", + dataset_type=DatasetType.INFORMATION_RETRIEVAL, + results=[ + FlexibleDatasetRow( + data={ + "query_id": query_id, + "chunk_id": "p1", + "paragraph": "retrieved para", + "relevant_part_id": "rel1", + "relevant_text_sim": 0.99, + "relevance_label": 2, + "report": "Report A", + "question": "What is X?", + "position": 1, + } + ), + FlexibleDatasetRow( + data={ + "query_id": query_id, + "chunk_id": "p2", + "paragraph": "other para", + "relevant_part_id": "rel1", + "relevant_text_sim": 0.5, + "relevance_label": "0", + "report": "Report A", + "question": "What is X?", + "position": 2, + } + ), + # Duplicate chunk+part keeps higher sim + FlexibleDatasetRow( + data={ + "query_id": query_id, + "chunk_id": "p1", + "paragraph": "retrieved para better", + "relevant_part_id": "rel1", + "relevant_text_sim": 0.995, + "relevance_label": 1, + "report": "Report A", + "question": "What is X?", + "position": 1, + } + ), + ], + ) + + df = build_error_analysis_dataframe_from_flexible(gt, bench, top_k=2) + + assert len(df) == 2 + assert df.iloc[0]["chunk_id"] == "p1" + assert bool(df.iloc[0]["is_really_relevant"]) is True + assert "gold relevant text" in df.iloc[0]["relevant_part_text"] + assert df.iloc[0]["position_in_top_k"] == 1 + + +def test_build_error_analysis_dataframe_from_flexible_skips_pairs_without_gt(): + bench = BenchmarkDataset( + dataset_id="bm", + name="BM", + results=[ + FlexibleDatasetRow( + data={ + "query_id": "R|||Q", + "chunk_id": "p1", + "paragraph": "x", + "relevant_text_sim": 0.9, + "report": "R", + "question": "Q", + } + ) + ], + ) + gt = BenchmarkDataset(dataset_id="gt", name="GT", results=[]) + df = build_error_analysis_dataframe_from_flexible(gt, bench, top_k=5) + assert df.empty + + +def test_build_error_analysis_dataframe_from_flexible_query_fallback(): + """When report/question cannot be parsed, fall back to query_id matching.""" + gt = BenchmarkDataset( + dataset_id="gt", + name="GT", + results=[ + FlexibleDatasetRow( + data={ + "query_id": "q_only", + "chunk_id": "c1", + "context": "gt text", + "score": 1.0, + } + ) + ], + ) + bench = BenchmarkDataset( + dataset_id="bm", + name="BM", + results=[ + FlexibleDatasetRow( + data={ + "query_id": "q_only", + "chunk_id": "c1", + "paragraph": "retrieved", + "score": 0.8, + } + ), + FlexibleDatasetRow( + data={ + "query_id": "q_only", + "chunk_id": "", # skipped in fallback + "paragraph": "no id", + "score": 0.7, + } + ), + ], + ) + + df = build_error_analysis_dataframe_from_flexible(gt, bench, top_k=3) + + assert len(df) == 1 + assert df.iloc[0]["chunk_id"] == "c1" + assert bool(df.iloc[0]["is_really_relevant"]) is True + assert df.iloc[0]["relevant_part_text"] == "gt text" + + +def test_build_error_analysis_dataframe_from_flexible_skips_incomplete_gt_rows(): + gt = BenchmarkDataset( + dataset_id="gt", + name="GT", + results=[ + FlexibleDatasetRow(data={"query_id": "", "chunk_id": "c1"}), + FlexibleDatasetRow(data={"query_id": "q1", "chunk_id": ""}), + ], + ) + bench = BenchmarkDataset( + dataset_id="bm", + name="BM", + results=[FlexibleDatasetRow(data={"query_id": "", "chunk_id": "x"})], + ) + df = build_error_analysis_dataframe_from_flexible(gt, bench, top_k=1) + assert df.empty + + +def test_build_error_analysis_dataframe_from_flexible_string_relevance_invalid(): + query_id = "Doc|||Ask" + gt = BenchmarkDataset( + dataset_id="gt", + name="GT", + results=[ + FlexibleDatasetRow( + data={ + "query_id": query_id, + "chunk_id": "rel", + "relevant": "gold", + "document": "Doc", + "question": "Ask", + "score": 1, + } + ) + ], + ) + bench = BenchmarkDataset( + dataset_id="bm", + name="BM", + results=[ + FlexibleDatasetRow( + data={ + "query_id": query_id, + "chunk_id": "p1", + "paragraph": "p", + "relevant_part_id": "other", + "relevant_text_sim": 0.2, + "relevance_label": "not-a-number", + "report": "Doc", + "question": "Ask", + } + ) + ], + ) + df = build_error_analysis_dataframe_from_flexible(gt, bench, top_k=1) + assert len(df) == 1 + assert bool(df.iloc[0]["is_really_relevant"]) is False + # Falls back to highest-score GT part for the report/question pair + assert df.iloc[0]["relevant_part_text"] == "gold" + + +def test_build_error_analysis_uses_simple_namespace_text_attr(): + """Legacy callers may pass objects with `.text` instead of question_text.""" + chunk = SimpleNamespace(chunk_id="c1", relevance_score=0.5, text="t", metadata={"report": "R"}) + question = SimpleNamespace(question_id="q1", text="Q?", ground_truth_chunks=[chunk]) + gt = SimpleNamespace(questions=[question]) + retrieval = [RetrievalResultRow(query_id="q1", chunk_id="c1", position=1, score=0.1)] + df = build_error_analysis_dataframe(gt, retrieval, top_k=1) # type: ignore[arg-type] + assert df.iloc[0]["question"] == "Q?" + assert df.iloc[0]["report_name"] == "R" diff --git a/tests/test_evaluation_engine_coverage.py b/tests/test_evaluation_engine_coverage.py new file mode 100644 index 00000000..de603ceb --- /dev/null +++ b/tests/test_evaluation_engine_coverage.py @@ -0,0 +1,104 @@ +"""Additional coverage for evaluation engine dataset comparison.""" + +import pytest + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.core.benchmark.retrieval_results_loader import load_flexible_dataset_from_csv +from report_analyst.models.benchmark import ( + EvaluationMetrics, + RetrievalResultRow, + RetrievalResultsDataset, +) + + +def _legacy_ds(dataset_id: str, rows): + return RetrievalResultsDataset( + dataset_id=dataset_id, + name=dataset_id, + source="csv", + results=rows, + ) + + +def test_compare_datasets_legacy_matching(): + engine = EvaluationEngine() + reference = _legacy_ds( + "ref", + [ + RetrievalResultRow(query_id="q1", chunk_id="c1", position=1, score=1.0), + RetrievalResultRow(query_id="q1", chunk_id="c2", position=2, score=0.5), + ], + ) + input_ds = _legacy_ds( + "inp", + [ + RetrievalResultRow( + query_id="q1", + chunk_id="c1", + position=1, + score=0.9, + metadata={"relevant_text_sim": 0.99}, + ), + RetrievalResultRow( + query_id="q1", + chunk_id="c99", + position=2, + score=0.8, + metadata={"relevant_text_sim": 0.1}, + ), + ], + ) + metrics = engine.compare_datasets(reference, input_ds, k_values=[1, 2]) + assert metrics.precision_at_k[1] > 0 + assert 2 in metrics.precision_at_k + + +def test_compare_datasets_no_common_queries(): + engine = EvaluationEngine() + reference = _legacy_ds( + "ref", + [RetrievalResultRow(query_id="q1", chunk_id="c1", position=1, score=1.0)], + ) + input_ds = _legacy_ds( + "inp", + [RetrievalResultRow(query_id="q2", chunk_id="c1", position=1, score=1.0)], + ) + metrics = engine.compare_datasets(reference, input_ds) + assert metrics.mean_average_precision == 0.0 + + +def test_compare_flexible_with_relevant_part_id_and_sim_scores(): + engine = EvaluationEngine() + reference_csv = """query_id,chunk_id,position,score,relevant_part_id +q1,gt1,1,1.0,gt1 +q1,gt2,2,0.5,gt2""" + input_csv = """query_id,chunk_id,position,score,relevant_part_id,relevant_text_sim +q1,p1,1,0.1,gt1,0.99 +q1,p2,2,0.1,gt2,0.5""" + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + metrics = engine.compare_flexible_datasets(reference, input_dataset, k_values=[1, 2]) + assert metrics.precision_at_k[1] == 1.0 + + +def test_compare_evaluations_reports_improvements(): + engine = EvaluationEngine() + e1 = EvaluationMetrics( + precision_at_k={1: 0.5}, + recall_at_k={1: 0.4}, + f1_at_k={1: 0.45}, + mean_reciprocal_rank=0.5, + mean_average_precision=0.4, + ndcg_at_k={1: 0.3}, + ) + e2 = EvaluationMetrics( + precision_at_k={1: 0.8}, + recall_at_k={1: 0.7}, + f1_at_k={1: 0.75}, + mean_reciprocal_rank=0.9, + mean_average_precision=0.8, + ndcg_at_k={1: 0.6}, + ) + diff = engine.compare_evaluations(e1, e2) + assert diff["precision_at_1_improvement"] == pytest.approx(0.3) + assert diff["mrr_improvement"] == pytest.approx(0.4) diff --git a/tests/test_external_service_integration.py b/tests/test_external_service_integration.py index cc997b5e..0ea64dc5 100644 --- a/tests/test_external_service_integration.py +++ b/tests/test_external_service_integration.py @@ -8,17 +8,11 @@ - Analysis request and result delivery """ -import json -import tempfile -from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, Mock, patch +from unittest.mock import AsyncMock, Mock, patch import pytest -from aioresponses import aioresponses -from report_analyst_search_backend.external_service_client import ( - ExternalServiceClient, -) +from report_analyst_search_backend.external_service_client import ExternalServiceClient from report_analyst_search_backend.external_service_delivery import ( ExternalServiceDelivery, ) @@ -204,10 +198,21 @@ def test_chunks_match_format(self, external_handler, sample_chunks): class TestExternalServiceClient: """Tests for ExternalServiceClient""" + def _mock_http_response(self, status=200, payload=None, text=""): + response = AsyncMock() + response.status = status + response.json = AsyncMock(return_value=payload or {}) + response.text = AsyncMock(return_value=text) + + context = AsyncMock() + context.__aenter__.return_value = response + context.__aexit__.return_value = None + return context, response + @pytest.mark.asyncio async def test_notify_ready_nats(self, external_client): """Test notifying via NATS""" - with patch.object(external_client, "connect_nats") as mock_connect: + with patch.object(external_client, "connect_nats"): with patch.object(external_client, "js") as mock_js: mock_js.publish = AsyncMock() external_client.nc = Mock() @@ -227,11 +232,14 @@ async def test_notify_ready_nats(self, external_client): @pytest.mark.asyncio async def test_notify_ready_http(self, external_client): """Test notifying via HTTP""" - with aioresponses() as m: - m.post( - "http://localhost:8000/external/services/service-x/notify", - status=200, - ) + response_context, _ = self._mock_http_response(status=200) + session = Mock() + session.post.return_value = response_context + session_context = AsyncMock() + session_context.__aenter__.return_value = session + session_context.__aexit__.return_value = None + + with patch("aiohttp.ClientSession", return_value=session_context): result = await external_client.notify_ready( service_id="service-x", request_id="req-123", @@ -240,16 +248,19 @@ async def test_notify_ready_http(self, external_client): method="http", ) assert result is True + session.post.assert_called_once() @pytest.mark.asyncio async def test_request_analysis_http(self, external_client): """Test requesting analysis via HTTP""" - with aioresponses() as m: - m.post( - "http://localhost:8000/external/services/service-x/analyze", - status=200, - payload={"request_id": "analysis-123"}, - ) + response_context, _ = self._mock_http_response(status=200, payload={"request_id": "analysis-123"}) + session = Mock() + session.post.return_value = response_context + session_context = AsyncMock() + session_context.__aenter__.return_value = session + session_context.__aexit__.return_value = None + + with patch("aiohttp.ClientSession", return_value=session_context): request_id = await external_client.request_analysis( service_id="service-x", external_request_id="req-123", @@ -259,25 +270,32 @@ async def test_request_analysis_http(self, external_client): method="http", ) assert request_id == "analysis-123" + session.post.assert_called_once() @pytest.mark.asyncio async def test_get_results(self, external_client): """Test polling for results""" - with aioresponses() as m: - m.get( - "http://localhost:8000/external/services/service-x/results/analysis-123", - status=200, - payload={ - "request_id": "analysis-123", - "status": "completed", - "answers": [], - "top_chunks": [], - }, - ) + response_context, _ = self._mock_http_response( + status=200, + payload={ + "request_id": "analysis-123", + "status": "completed", + "answers": [], + "top_chunks": [], + }, + ) + session = Mock() + session.get.return_value = response_context + session_context = AsyncMock() + session_context.__aenter__.return_value = session + session_context.__aexit__.return_value = None + + with patch("aiohttp.ClientSession", return_value=session_context): results = await external_client.get_results("service-x", "analysis-123") assert results is not None assert results["status"] == "completed" + session.get.assert_called_once() class TestExternalServiceDelivery: @@ -286,7 +304,7 @@ class TestExternalServiceDelivery: @pytest.mark.asyncio async def test_deliver_results_nats(self, external_delivery): """Test delivering results via NATS""" - with patch.object(external_delivery, "connect_nats") as mock_connect: + with patch.object(external_delivery, "connect_nats"): with patch.object(external_delivery, "js") as mock_js: mock_js.publish = AsyncMock() external_delivery.nc = Mock() diff --git a/tests/test_flexible_alignment.py b/tests/test_flexible_alignment.py new file mode 100644 index 00000000..28586410 --- /dev/null +++ b/tests/test_flexible_alignment.py @@ -0,0 +1,142 @@ +"""Unit tests for flexible benchmark dataset alignment.""" + +import pandas as pd +import pytest + +from report_analyst.core.benchmark.dataset_mapper import ( + generate_chunk_id, + generate_query_id, +) +from report_analyst.core.benchmark.flexible_alignment import ( + BenchmarkAlignConfig, + GroundTruthAlignConfig, + align_benchmark_flexible, + align_ground_truth_flexible, +) + + +class TestAlignGroundTruthFlexible: + """Tests for align_ground_truth_flexible.""" + + @pytest.fixture + def raw_ground_truth(self): + return pd.DataFrame( + { + "report": ["doc_a", "doc_a"], + "question": ["What is scope 1?", "What is scope 1?"], + "chunk": ["Scope 1 emissions data", "Governance overview"], + "relevance": [2, 0], + } + ) + + def test_aligns_required_columns_and_derives_ids(self, raw_ground_truth): + config = GroundTruthAlignConfig( + document_col="report", + question_col="question", + chunk_text_col="chunk", + relevant_part_col=None, + label_cols=["relevance"], + ) + + aligned = align_ground_truth_flexible(raw_ground_truth, config) + + assert list(aligned.columns) == [ + "query_id", + "chunk_id", + "relevant_part_id", + "question", + "document", + "chunk_text", + "relevant_part_text", + "relevance", + "score", + ] + assert aligned.loc[0, "query_id"] == generate_query_id("doc_a", "What is scope 1?") + assert aligned.loc[0, "chunk_id"] == generate_chunk_id("Scope 1 emissions data") + assert aligned.loc[0, "relevant_part_id"] == aligned.loc[0, "chunk_id"] + assert aligned.loc[0, "score"] == 2.0 + + def test_raises_when_label_column_missing(self, raw_ground_truth): + config = GroundTruthAlignConfig( + document_col="report", + question_col="question", + chunk_text_col="chunk", + relevant_part_col=None, + label_cols=["missing_label"], + ) + + with pytest.raises(ValueError, match="Label column 'missing_label'"): + align_ground_truth_flexible(raw_ground_truth, config) + + +class TestAlignBenchmarkFlexible: + """Tests for align_benchmark_flexible.""" + + @pytest.fixture + def raw_benchmark(self): + return pd.DataFrame( + { + "query_id": ["q1", "q1"], + "chunk_text": ["Alpha chunk", "Beta chunk"], + "pred_relevance": [2, 1], + "relevant_text_sim": [0.9, 0.4], + } + ) + + def test_reuses_existing_query_id_and_deduplicates_ranking_column(self, raw_benchmark): + config = BenchmarkAlignConfig( + document_col=None, + question_col=None, + query_id_col="query_id", + chunk_text_col="chunk_text", + relevant_part_col=None, + prediction_cols=["pred_relevance", "relevant_text_sim"], + ranking_score_col="relevant_text_sim", + ) + + aligned = align_benchmark_flexible(raw_benchmark, config) + + assert aligned["query_id"].tolist() == ["q1", "q1"] + assert aligned["chunk_id"].tolist() == [ + generate_chunk_id("Alpha chunk"), + generate_chunk_id("Beta chunk"), + ] + assert "relevant_text_sim" in aligned.columns + assert aligned.columns.tolist().count("relevant_text_sim") == 1 + + def test_builds_query_id_from_question_when_missing_query_id_column(self): + raw = pd.DataFrame( + { + "report": ["doc_a"], + "question": ["What is scope 1?"], + "chunk_text": ["Scope 1 emissions data"], + "pred_relevance": [2], + } + ) + config = BenchmarkAlignConfig( + document_col="report", + question_col="question", + query_id_col=None, + chunk_text_col="chunk_text", + relevant_part_col=None, + prediction_cols=["pred_relevance"], + ranking_score_col=None, + ) + + aligned = align_benchmark_flexible(raw, config) + + assert aligned.loc[0, "query_id"] == generate_query_id("doc_a", "What is scope 1?") + + def test_raises_when_neither_query_id_nor_question_provided(self, raw_benchmark): + config = BenchmarkAlignConfig( + document_col=None, + question_col=None, + query_id_col=None, + chunk_text_col="chunk_text", + relevant_part_col=None, + prediction_cols=["pred_relevance"], + ranking_score_col=None, + ) + + with pytest.raises(ValueError, match="Either an explicit query_id_col"): + align_benchmark_flexible(raw_benchmark.drop(columns=["query_id"]), config) diff --git a/tests/test_flexible_dataset_evaluation.py b/tests/test_flexible_dataset_evaluation.py new file mode 100644 index 00000000..91a6a03d --- /dev/null +++ b/tests/test_flexible_dataset_evaluation.py @@ -0,0 +1,142 @@ +import pytest + +from report_analyst.core.benchmark.evaluation_engine import EvaluationEngine +from report_analyst.core.benchmark.retrieval_results_loader import ( + load_flexible_dataset_from_csv, +) + + +class TestFlexibleDatasetEvaluation: + """Test suite for evaluating flexible datasets""" + + @pytest.fixture + def engine(self): + return EvaluationEngine() + + def test_compare_ir_datasets_basic(self, engine): + """Test comparing two IR datasets with matching chunks""" + reference_csv = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,1.0 +tcfd_1,chunk_015,2,0.9 +tcfd_2,chunk_023,1,1.0""" + + input_csv = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89 +tcfd_2,chunk_023,1,0.93""" + + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + + metrics = engine.compare_flexible_datasets(reference, input_dataset) + + assert metrics is not None + assert 1 in metrics.precision_at_k + assert metrics.precision_at_k[1] > 0 + + def test_compare_ir_datasets_partial_match(self, engine): + """Test comparing IR datasets with partial chunk matches""" + reference_csv = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,1.0 +tcfd_1,chunk_015,2,0.9 +tcfd_1,chunk_042,3,0.8""" + + input_csv = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_999,2,0.89 +tcfd_1,chunk_015,3,0.87""" + + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + + metrics = engine.compare_flexible_datasets(reference, input_dataset, k_values=[1, 2, 3]) + + assert metrics.precision_at_k[1] == 1.0 + assert metrics.precision_at_k[2] < 1.0 + assert metrics.precision_at_k[3] < 1.0 + + def test_compare_ir_datasets_no_common_queries(self, engine): + """Test comparing datasets with no common queries""" + reference_csv = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,1.0""" + + input_csv = """query_id,chunk_id,position,score +tcfd_99,chunk_001,1,0.95""" + + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + + metrics = engine.compare_flexible_datasets(reference, input_dataset) + + assert metrics.mean_average_precision == 0.0 + assert metrics.mean_reciprocal_rank == 0.0 + + def test_compare_ie_datasets_exact_match(self, engine): + """Test comparing IE datasets with exact answer matches""" + reference_csv = """question_id,answer,category +tcfd_1,"The company identifies climate risks","risk_identification" +tcfd_2,"Strategy includes climate considerations","strategy" """ + + input_csv = """question_id,answer,category +tcfd_1,"The company identifies climate risks","risk_identification" +tcfd_2,"Strategy includes climate considerations","strategy" """ + + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + + metrics = engine.compare_flexible_datasets(reference, input_dataset) + + assert metrics.precision_at_k[1] == 1.0 + assert metrics.mean_average_precision == 1.0 + + def test_compare_ie_datasets_partial_match(self, engine): + """Test comparing IE datasets with partial matches""" + reference_csv = """question_id,answer +tcfd_1,"The company identifies climate risks" +tcfd_2,"Strategy includes climate considerations" """ + + input_csv = """question_id,answer +tcfd_1,"The company identifies climate risks" +tcfd_2,"Different answer here" """ + + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + + metrics = engine.compare_flexible_datasets(reference, input_dataset) + + assert metrics.precision_at_k[1] == 0.5 + assert metrics.mean_average_precision == 0.5 + + def test_compare_datasets_with_variant_column_names(self, engine): + """Test comparing datasets with different column name variations""" + reference_csv = """query_id,chunk_id,position,relevance_score +tcfd_1,chunk_001,1,1.0 +tcfd_1,chunk_015,2,0.9""" + + input_csv = """question_id,chunk,rank,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89""" + + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + + metrics = engine.compare_flexible_datasets(reference, input_dataset) + + assert metrics is not None + assert len(reference.get_unique_queries()) == 1 + assert len(input_dataset.get_unique_queries()) == 1 + + def test_compare_datasets_different_types_warning(self, engine): + """Test that comparing different dataset types shows warning but still works""" + reference_csv = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,1.0""" + + input_csv = """question_id,answer +tcfd_1,"Some answer" """ + + reference = load_flexible_dataset_from_csv(csv_content=reference_csv) + input_dataset = load_flexible_dataset_from_csv(csv_content=input_csv) + + # Should not raise, but may show warning + metrics = engine.compare_flexible_datasets(reference, input_dataset) + assert metrics is not None diff --git a/tests/test_llm_models.py b/tests/test_llm_models.py new file mode 100644 index 00000000..fbf6c438 --- /dev/null +++ b/tests/test_llm_models.py @@ -0,0 +1,73 @@ +"""Tests for env-configurable LLM model lists.""" + +import pytest + +import report_analyst.core.llm_models as llm_models + + +@pytest.fixture(autouse=True) +def _clear_model_env(monkeypatch): + for key in ("OPENAI_MODELS", "GEMINI_MODELS", "LLM_MODELS", "OPENAI_API_MODEL", "DEFAULT_MODEL", "GOOGLE_API_KEY"): + monkeypatch.delenv(key, raising=False) + + +def test_default_openai_models_include_gpt_54(): + models = llm_models.get_openai_models() + assert models[0] == "gpt-5.4-mini" + assert "gpt-5.4" in models + assert "gpt-4o-mini" in models + + +def test_default_gemini_models_include_latest(): + models = llm_models.get_gemini_models() + assert models[0] == "gemini-3.5-flash" + assert "gemini-2.5-flash" in models + assert "gemini-1.5-flash" in models + + +def test_openai_models_env_override(monkeypatch): + monkeypatch.setenv("OPENAI_MODELS", "gpt-5.4-mini,custom-model") + assert llm_models.get_openai_models() == ["gpt-5.4-mini", "custom-model"] + + +def test_gemini_models_env_override(monkeypatch): + monkeypatch.setenv("GEMINI_MODELS", "gemini-3.5-flash") + assert llm_models.get_gemini_models() == ["gemini-3.5-flash"] + + +def test_llm_models_combined_env_override(monkeypatch): + monkeypatch.setenv("LLM_MODELS", "gpt-5.4,gemini-3.5-flash") + assert llm_models.get_llm_models(include_gemini=False) == ["gpt-5.4", "gemini-3.5-flash"] + + +def test_llm_models_merges_gemini_when_key_set(monkeypatch): + monkeypatch.setenv("GOOGLE_API_KEY", "test-key") + models = llm_models.get_llm_models() + assert models[0].startswith("gpt-") + assert any(m.startswith("gemini-") for m in models) + + +def test_llm_models_omits_gemini_without_key(monkeypatch): + models = llm_models.get_llm_models() + assert all(not m.startswith("gemini-") for m in models) + + +def test_default_llm_model_from_openai_api_model(monkeypatch): + monkeypatch.setenv("OPENAI_API_MODEL", "gpt-5.4-mini") + assert llm_models.get_default_llm_model() == "gpt-5.4-mini" + + +def test_default_llm_model_falls_back_to_first_list_entry(monkeypatch): + assert llm_models.get_default_llm_model() == "gpt-5.4-mini" + + +def test_models_for_api_shape(monkeypatch): + rows = llm_models.get_models_for_api(include_gemini=False) + assert rows[0]["id"] == "gpt-5.4-mini" + assert "GPT" in rows[0]["name"] + + +def test_models_for_api_respects_openai_env(monkeypatch): + monkeypatch.setenv("OPENAI_MODELS", "gpt-5.4-mini,gpt-4o-mini") + rows = llm_models.get_models_for_api(include_gemini=False) + assert [row["id"] for row in rows] == ["gpt-5.4-mini", "gpt-4o-mini"] diff --git a/tests/test_question_loader.py b/tests/test_question_loader.py index cb63b064..d1d9eb7f 100644 --- a/tests/test_question_loader.py +++ b/tests/test_question_loader.py @@ -5,7 +5,6 @@ import os import tempfile -import pytest import yaml from report_analyst.core.question_loader import ( @@ -129,7 +128,7 @@ def test_get_question_set_options(self): """Test get_question_set_options method returns list of question set IDs""" with tempfile.TemporaryDirectory() as temp_dir: # Create multiple test question set files - for i, name in enumerate(["test1", "test2", "test3"]): + for _i, name in enumerate(["test1", "test2", "test3"]): test_questions = { "name": f"{name.title()} Questions", "shortcut": name, @@ -184,7 +183,15 @@ def test_load_real_question_sets(self): question_sets = loader.get_question_sets() # Should have at least the main question sets - expected_sets = ["everest", "tcfd", "denali", "kilimanjaro"] + # "custom" is a UI-only option and not a real question set ID + expected_sets = [ + "everest", + "tcfd", + "denali", + "kilimanjaro", + "lucia", + "climretrieve", + ] for expected_set in expected_sets: assert expected_set in question_sets, f"Expected {expected_set} in question sets" @@ -301,7 +308,7 @@ def test_fallback_behavior_without_core_functionality(self): from report_analyst.core.question_loader import get_question_loader question_loader = get_question_loader() - question_set_options = question_loader.get_question_set_options() + ["custom"] + question_set_options = [*question_loader.get_question_set_options(), "custom"] else: # Fallback: use a generic approach without hardcoded names question_set_options = ["custom"] # Only custom when core functionality unavailable @@ -319,15 +326,24 @@ def test_normal_behavior_with_core_functionality(self): from report_analyst.core.question_loader import get_question_loader question_loader = get_question_loader() - question_set_options = question_loader.get_question_set_options() + ["custom"] + question_set_options = [*question_loader.get_question_set_options(), "custom"] else: # Fallback: use a generic approach without hardcoded names question_set_options = ["custom"] # Only custom when core functionality unavailable # Should have all question sets plus custom - expected_sets = {"everest", "tcfd", "denali", "kilimanjaro", "lucia", "custom"} + expected_sets = { + "everest", + "tcfd", + "denali", + "kilimanjaro", + "lucia", + "climretrieve", + "custom", + } assert set(question_set_options) == expected_sets - assert len(question_set_options) == 6 + # 6 real sets + "custom" UI option + assert len(question_set_options) == 7 assert "custom" in question_set_options assert "everest" in question_set_options assert "tcfd" in question_set_options @@ -371,8 +387,15 @@ def test_question_set_options_consistency(self): assert options1 == options2 == options3 # Should contain expected question sets - expected_sets = ["everest", "tcfd", "denali", "kilimanjaro", "lucia"] + expected_sets = [ + "everest", + "tcfd", + "denali", + "kilimanjaro", + "lucia", + "climretrieve", + ] for expected_set in expected_sets: assert expected_set in options1, f"Expected question set '{expected_set}' not found in options" - assert len(options1) == 5 + assert len(options1) == 6 diff --git a/tests/test_retrieval_results_loader.py b/tests/test_retrieval_results_loader.py new file mode 100644 index 00000000..8ad5304b --- /dev/null +++ b/tests/test_retrieval_results_loader.py @@ -0,0 +1,290 @@ +import sqlite3 +import tempfile +from pathlib import Path + +import pandas as pd +import pytest + +from report_analyst.core.benchmark.retrieval_results_loader import ( + detect_dataset_type, + export_retrieval_results_to_csv, + load_flexible_dataset_from_csv, + load_flexible_dataset_from_sqlite, +) +from report_analyst.models.benchmark import ( + BenchmarkDataset, + DatasetType, + FlexibleDatasetRow, +) + + +class TestFlexibleDatasetLoader: + """Test suite for flexible dataset loading with different column names""" + + def test_detect_dataset_type_ir(self): + """Test detecting Information Retrieval dataset type""" + columns = ["query_id", "chunk_id", "position", "score"] + dataset_type = detect_dataset_type(columns) + assert dataset_type == DatasetType.INFORMATION_RETRIEVAL + + def test_detect_dataset_type_ie(self): + """Test detecting Information Extraction dataset type""" + columns = ["question_id", "answer", "category", "confidence_score"] + dataset_type = detect_dataset_type(columns) + assert dataset_type == DatasetType.INFORMATION_EXTRACTION + + def test_detect_dataset_type_defaults_to_ir(self): + """Test that unclear datasets default to IR""" + columns = ["id", "value"] + dataset_type = detect_dataset_type(columns) + assert dataset_type == DatasetType.INFORMATION_RETRIEVAL + + def test_load_csv_with_standard_columns(self): + """Test loading CSV with standard column names""" + csv_content = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89 +tcfd_2,chunk_023,1,0.93""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + assert dataset.dataset_id is not None + assert dataset.dataset_type == DatasetType.INFORMATION_RETRIEVAL + assert len(dataset.results) == 3 + + def test_load_csv_with_variant_column_names(self): + """Test loading CSV with variant column names (question_id instead of query_id)""" + csv_content = """question_id,chunk,rank,relevance_score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + assert len(dataset.results) == 2 + assert dataset.results[0].get_query_id() == "tcfd_1" + assert dataset.results[0].get_chunk_id() == "chunk_001" + assert dataset.results[0].get_position() == 1 + assert dataset.results[0].get_score() == 0.95 + + def test_load_csv_ie_dataset(self): + """Test loading Information Extraction dataset""" + csv_content = """question_id,answer,category,confidence_score +tcfd_1,"The company identifies climate risks...","risk_identification",0.92 +tcfd_2,"Strategy includes...","strategy",0.88""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + assert dataset.dataset_type == DatasetType.INFORMATION_EXTRACTION + assert len(dataset.results) == 2 + assert dataset.results[0].get_answer() == "The company identifies climate risks..." + assert dataset.results[0].get_category() == "risk_identification" + + def test_load_csv_from_file_path(self): + """Test loading CSV from file path""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f: + f.write("query_id,chunk_id,position,score\n") + f.write("tcfd_1,chunk_001,1,0.95\n") + f.flush() + temp_path = f.name + + try: + dataset = load_flexible_dataset_from_csv(csv_path=temp_path) + assert len(dataset.results) == 1 + assert dataset.source_path == temp_path + finally: + Path(temp_path).unlink() + + def test_load_csv_with_optional_columns(self): + """Test loading CSV with optional columns""" + csv_content = """query_id,report_id,chunk_id,chunk_text,position,score,similarity_score,llm_score +tcfd_1,report_001,chunk_001,"Climate risks...",1,0.95,0.92,0.88 +tcfd_1,report_001,chunk_015,"Risk assessment...",2,0.89,0.87,0.85""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + assert len(dataset.results) == 2 + assert dataset.results[0].get("report_id") == "report_001" + assert dataset.results[0].get("chunk_text") == "Climate risks..." + assert dataset.results[0].get("similarity_score") == 0.92 + assert dataset.results[0].get("llm_score") == 0.88 + + def test_load_csv_handles_missing_values(self): + """Test loading CSV with missing/NaN values""" + csv_content = """query_id,chunk_id,position,score,similarity_score +tcfd_1,chunk_001,1,0.95, +tcfd_1,chunk_015,2,0.89,0.87""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + assert len(dataset.results) == 2 + assert dataset.results[0].get("similarity_score") is None + assert dataset.results[1].get("similarity_score") == 0.87 + + def test_load_csv_requires_query_id(self): + """Test that CSV must have query_id or question_id""" + csv_content = """chunk_id,position,score +chunk_001,1,0.95""" + + with pytest.raises(ValueError, match="Missing required columns"): + load_flexible_dataset_from_csv(csv_content=csv_content) + + def test_load_csv_requires_chunk_id_for_ir(self): + """Test that IR datasets require chunk_id""" + csv_content = """query_id,position,score +tcfd_1,1,0.95""" + + with pytest.raises(ValueError, match="Missing required columns"): + load_flexible_dataset_from_csv(csv_content=csv_content) + + def test_get_unique_queries(self): + """Test getting unique query IDs from dataset""" + csv_content = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89 +tcfd_2,chunk_023,1,0.93""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + unique_queries = dataset.get_unique_queries() + assert len(unique_queries) == 2 + assert "tcfd_1" in unique_queries + assert "tcfd_2" in unique_queries + + def test_get_results_by_query(self): + """Test getting results for a specific query""" + csv_content = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89 +tcfd_2,chunk_023,1,0.93""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + tcfd_1_results = dataset.get_results_by_query("tcfd_1") + assert len(tcfd_1_results) == 2 + assert tcfd_1_results[0].get_chunk_id() == "chunk_001" + assert tcfd_1_results[1].get_chunk_id() == "chunk_015" + + def test_is_metadata_only(self): + """Test checking if dataset is metadata-only""" + dataset = BenchmarkDataset(dataset_id="test", name="Test", source="database") + assert dataset.is_metadata_only() is True + + def test_has_results(self): + """Test checking if dataset has results""" + csv_content = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + assert dataset.has_results() is True + + def test_load_sqlite_dataset(self): + """Test loading dataset from SQLite""" + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp_db: + db_path = tmp_db.name + + try: + # Create table and insert data + conn = sqlite3.connect(db_path) + conn.execute( + """ + CREATE TABLE benchmark_results ( + query_id TEXT, + chunk_id TEXT, + position INTEGER, + score REAL + ) + """ + ) + conn.execute( + """ + INSERT INTO benchmark_results VALUES + ('tcfd_1', 'chunk_001', 1, 0.95), + ('tcfd_1', 'chunk_015', 2, 0.89) + """ + ) + conn.commit() + conn.close() + + dataset = load_flexible_dataset_from_sqlite(db_path, table_name="benchmark_results") + + assert len(dataset.results) == 2 + assert dataset.source == "sqlite" + assert dataset.source_path == db_path + finally: + Path(db_path).unlink() + + def test_load_sqlite_with_filter(self): + """Test loading SQLite dataset with query filter""" + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp_db: + db_path = tmp_db.name + + try: + conn = sqlite3.connect(db_path) + conn.execute( + """ + CREATE TABLE benchmark_results ( + query_id TEXT, + chunk_id TEXT, + position INTEGER, + score REAL + ) + """ + ) + conn.execute( + """ + INSERT INTO benchmark_results VALUES + ('tcfd_1', 'chunk_001', 1, 0.95), + ('tcfd_2', 'chunk_023', 1, 0.93) + """ + ) + conn.commit() + conn.close() + + dataset = load_flexible_dataset_from_sqlite( + db_path, + table_name="benchmark_results", + query_filter="query_id = 'tcfd_1'", + ) + + assert len(dataset.results) == 1 + assert dataset.results[0].get_query_id() == "tcfd_1" + finally: + Path(db_path).unlink() + + def test_export_to_csv(self): + """Test exporting dataset to CSV""" + csv_content = """query_id,chunk_id,position,score +tcfd_1,chunk_001,1,0.95 +tcfd_1,chunk_015,2,0.89""" + + dataset = load_flexible_dataset_from_csv(csv_content=csv_content) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f: + output_path = f.name + + try: + export_retrieval_results_to_csv(dataset, output_path) + + # Verify file was created + assert Path(output_path).exists() + + # Verify content + df = pd.read_csv(output_path) + assert len(df) == 2 + assert "query_id" in df.columns + assert "chunk_id" in df.columns + assert "position" in df.columns + assert "score" in df.columns + finally: + Path(output_path).unlink() + + def test_flexible_row_get_method(self): + """Test FlexibleDatasetRow get method with case-insensitive matching""" + row = FlexibleDatasetRow(data={"Query_ID": "tcfd_1", "Chunk_ID": "chunk_001", "Score": 0.95}) + + assert row.get("query_id") == "tcfd_1" + assert row.get("QUERY_ID") == "tcfd_1" + assert row.get("chunk_id") == "chunk_001" + assert row.get("score") == 0.95 + assert row.get("nonexistent") is None + assert row.get("nonexistent", "default") == "default" diff --git a/tests/test_retrieval_results_loader_coverage.py b/tests/test_retrieval_results_loader_coverage.py new file mode 100644 index 00000000..32a37a00 --- /dev/null +++ b/tests/test_retrieval_results_loader_coverage.py @@ -0,0 +1,109 @@ +"""Additional coverage for retrieval results loader.""" + +import sqlite3 +import tempfile +from pathlib import Path + +import pandas as pd +import pytest + +from report_analyst.core.benchmark.retrieval_results_loader import ( + export_retrieval_results_to_csv, + load_flexible_dataset_from_normalized_df, + load_retrieval_results_from_csv, + load_retrieval_results_from_sqlite, +) +from report_analyst.models.benchmark import ( + BenchmarkDataset, + FlexibleDatasetRow, +) + + +def test_load_retrieval_results_from_csv_content_and_bytes(): + csv_text = "query_id,chunk_id,position,score\nq1,c1,1,0.9\n" + ds = load_retrieval_results_from_csv(csv_content=csv_text, dataset_name="up") + assert len(ds.results) == 1 + assert ds.results[0].query_id == "q1" + + ds_b = load_retrieval_results_from_csv(csv_content=csv_text.encode("utf-8")) + assert len(ds_b.results) == 1 + + +def test_load_retrieval_results_from_csv_file_and_optional_cols(): + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f: + f.write("query_id,report_id,chunk_id,chunk_text,position,score,similarity_score,llm_score,extra\n") + f.write("q1,r1,c1,hello,1,0.9,0.8,0.7,x\n") + path = f.name + try: + ds = load_retrieval_results_from_csv(csv_path=path, dataset_id="fixed") + assert ds.dataset_id == "fixed" + assert ds.results[0].report_id == "r1" + assert ds.results[0].similarity_score == 0.8 + assert ds.results[0].metadata.get("extra") == "x" + finally: + Path(path).unlink() + + +def test_load_retrieval_results_from_csv_errors(tmp_path): + with pytest.raises(ValueError, match="Either csv_path"): + load_retrieval_results_from_csv() + with pytest.raises(FileNotFoundError): + load_retrieval_results_from_csv(csv_path=str(tmp_path / "does-not-exist-ra.csv")) + with pytest.raises(ValueError, match="Missing required"): + load_retrieval_results_from_csv(csv_content="a,b\n1,2\n") + + +def test_load_retrieval_results_from_sqlite_and_export(): + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + db_path = tmp.name + try: + conn = sqlite3.connect(db_path) + conn.execute( + """ + CREATE TABLE retrieval_results ( + query_id TEXT, report_id TEXT, chunk_id TEXT, chunk_text TEXT, + position INTEGER, score REAL, similarity_score REAL, llm_score REAL + ) + """ + ) + conn.execute("INSERT INTO retrieval_results VALUES ('q1','r1','c1','t',1,0.9,0.8,0.7)") + conn.commit() + conn.close() + + ds = load_retrieval_results_from_sqlite(db_path, dataset_name="sql") + assert len(ds.results) == 1 + assert ds.results[0].chunk_id == "c1" + assert ds.results[0].query_id == "q1" + finally: + Path(db_path).unlink(missing_ok=True) + + +def test_load_flexible_dataset_from_normalized_df(): + df = pd.DataFrame( + { + "query_id": ["q1"], + "chunk_id": ["c1"], + "position": [1], + "score": [0.5], + "paragraph": ["p"], + } + ) + ds = load_flexible_dataset_from_normalized_df(df, dataset_id="n1", dataset_name="norm") + assert ds.dataset_id == "n1" + assert len(ds.results) == 1 + assert ds.results[0].get_chunk_id() == "c1" + + with pytest.raises(ValueError, match="Missing"): + load_flexible_dataset_from_normalized_df(pd.DataFrame({"query_id": [1]})) + + +def test_export_retrieval_results_to_csv_accepts_benchmark_dataset(): + ds = BenchmarkDataset( + dataset_id="b", + name="b", + results=[FlexibleDatasetRow(data={"query_id": "q1", "chunk_id": "c1", "position": 1, "score": 0.1})], + ) + out = Path(tempfile.mkdtemp()) / "flex.csv" + export_retrieval_results_to_csv(ds, str(out)) + assert out.exists() + assert "q1" in out.read_text()