fix/test: increase LLM test coverage and fix validation bugs

classsify_extract.py → classify_extract.py

@@ -139,7 +139,6 @@ def _process_single_pdf(
     print(f"  [INFO] {pdf_path.name}: {len(original_text)} chars", file=sys.stderr)
 
     # ── Step 2: Classify ──────────────────────────────────────────────
-    clf_model, vectorizer, encoder = load_classifier(model_dir)
     label, confidence, pred_prob = classify_text(
         text=original_text,
         model=clf_model,

src/llm/llm_text.py

@@ -411,12 +411,12 @@ def load_document(file_path: Path) -> str:
     suffix = file_path.suffix.lower()
 
     if suffix == '.pdf':
-        print(f"[INFO] Reading PDF file...", file=sys.stderr)
+        print("[INFO] Reading PDF file...", file=sys.stderr)
         from src.preprocessing.pdf_text_extraction import extract_text_from_pdf
 
         return extract_text_from_pdf(str(file_path))
     elif suffix in ['.txt', '.text']:
-        print(f"[INFO] Reading text file...", file=sys.stderr)
+        print("[INFO] Reading text file...", file=sys.stderr)
         try:
             with open(file_path, "r", encoding="utf-8") as f:
                 return f.read()

src/llm/models.py

@@ -1,7 +1,7 @@
 """Pydantic models for predator diet data extraction."""
 
 from typing import Annotated, Optional
-from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt, computed_field, constr, model_validator
+from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt, computed_field, model_validator
 
 
 class PredatorDietMetrics(BaseModel):
@@ -20,13 +20,11 @@ class PredatorDietMetrics(BaseModel):
         frozen=False,
     )
 
-    species_name: Optional[
-        Annotated[
-            str,
-            constr(min_length=3, max_length=200, pattern=r"^[A-Z][a-z]+(\s[a-z]+)*$"),
-        ]
-    ] = Field(
+    species_name: Optional[str] = Field(
         default=None,
+        min_length=3,
+        max_length=200,
+        pattern=r"^[A-Z][a-z]+(\s[a-z]+)*$",
         description=(
             "Binomial scientific name of the PRIMARY PREDATOR species studied "
             "(the animal whose stomachs were examined, not its prey). "
@@ -37,8 +35,10 @@ class PredatorDietMetrics(BaseModel):
         examples=["Canis lupus", "Vulpes vulpes", "Pygoscelis papua", "Ursus arctos"],
     )
 
-    study_location: Optional[Annotated[str, constr(min_length=1, max_length=500)]] = Field(
+    study_location: Optional[str] = Field(
         default=None,
+        min_length=1,
+        max_length=500,
         description=(
             "Geographic area where predator specimens were collected. "
             "Include site, region, and country if available. "
@@ -48,17 +48,11 @@ class PredatorDietMetrics(BaseModel):
         examples=["Yellowstone National Park, Wyoming, USA", "Marion Island, sub-Antarctic", "Bristol, UK"],
     )
 
-    study_date: Optional[
-        Annotated[
-            str,
-            constr(
-                min_length=4,
-                max_length=30,
-                pattern=r"^\d{4}([\-—]\d{4})?$",
-            ),
-        ]
-    ] = Field(
+    study_date: Optional[str] = Field(
         default=None,
+        min_length=4,
+        max_length=30,
+        pattern=r"^\d{4}([\-—]\d{4})?$",
         description=(
             "Year or year-range when specimens were COLLECTED (not publication year). "
             "Format: 'YYYY' for single year or 'YYYY-YYYY' for range (e.g., '2019' or '2019-2021'). "

src/model/train_model.py

@@ -10,7 +10,6 @@
 import sys
 from pathlib import Path
 from collections import Counter
-from pathlib import Path
 
 import joblib
 import json

src/preprocessing/pdf_text_extraction.py

@@ -227,7 +227,7 @@ def extract_text_from_pdf_bytes(data: bytes, skip_ocr: bool = False) -> str:
         return ""
 
     if not skip_ocr and check_spelling(text) > MAX_SPELLING_ERROR_RATE:
-        print(f"[INFO] High misspelling rate — falling back to OCR", file=sys.stderr)
+        print("[INFO] High misspelling rate — falling back to OCR", file=sys.stderr)
         log.warning("High spelling error rate in in-memory PDF — falling back to OCR extraction.")
         text = []
         try:

tests/test_data_loader.py

@@ -18,8 +18,8 @@ def test_load_processed_text_reads_files(tmp_path):
     texts = load_processed_text(str(data_dir))
 
     assert len(texts) == 2
-    assert "example 1" in texts[0]
-    assert "example 2" in texts[1]
+    assert any("example 1" in t for t in texts)
+    assert any("example 2" in t for t in texts)
 
 
 def test_load_processed_text_empty_directory(tmp_path):