From 8bbd3e0ae8894d4ebcd8fb1f21ee4c1fea25d590 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 17:53:53 -0700 Subject: [PATCH 1/9] fix: rename classsify_extract.py to classify_extract.py --- classsify_extract.py => classify_extract.py | 1 - 1 file changed, 1 deletion(-) rename classsify_extract.py => classify_extract.py (99%) diff --git a/classsify_extract.py b/classify_extract.py similarity index 99% rename from classsify_extract.py rename to classify_extract.py index 4cdaa9f..f44fbbb 100644 --- a/classsify_extract.py +++ b/classify_extract.py @@ -139,7 +139,6 @@ def _process_single_pdf( print(f" [INFO] {pdf_path.name}: {len(original_text)} chars", file=sys.stderr) # ── Step 2: Classify ────────────────────────────────────────────── - clf_model, vectorizer, encoder = load_classifier(model_dir) label, confidence, pred_prob = classify_text( text=original_text, model=clf_model, From 08f2297458df0e8b6509723d8550c6cc40926f3c Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 17:53:58 -0700 Subject: [PATCH 2/9] fix(llm): fix Pydantic v2 string constraints not being enforced --- src/llm/llm_client.py | 1 - src/llm/models.py | 30 ++++++++++++------------------ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/llm/llm_client.py b/src/llm/llm_client.py index cd25900..dcbea52 100644 --- a/src/llm/llm_client.py +++ b/src/llm/llm_client.py @@ -147,7 +147,6 @@ def extract_metrics_from_text( if not _retry and missing: print( - f" [INFO] Retry: {', '.join(missing)} came back null — re-prompting", file=sys.stderr, ) diff --git a/src/llm/models.py b/src/llm/models.py index 9d05a2d..1810dac 100644 --- a/src/llm/models.py +++ b/src/llm/models.py @@ -1,7 +1,7 @@ """Pydantic models for predator diet data extraction.""" from typing import Annotated, Optional -from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt, computed_field, constr, model_validator +from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt, computed_field, model_validator class PredatorDietMetrics(BaseModel): @@ -20,13 +20,11 @@ class PredatorDietMetrics(BaseModel): frozen=False, ) - species_name: Optional[ - Annotated[ - str, - constr(min_length=3, max_length=200, pattern=r"^[A-Z][a-z]+(\s[a-z]+)*$"), - ] - ] = Field( + species_name: Optional[str] = Field( default=None, + min_length=3, + max_length=200, + pattern=r"^[A-Z][a-z]+(\s[a-z]+)*$", description=( "Binomial scientific name of the PRIMARY PREDATOR species studied " "(the animal whose stomachs were examined, not its prey). " @@ -37,8 +35,10 @@ class PredatorDietMetrics(BaseModel): examples=["Canis lupus", "Vulpes vulpes", "Pygoscelis papua", "Ursus arctos"], ) - study_location: Optional[Annotated[str, constr(min_length=1, max_length=500)]] = Field( + study_location: Optional[str] = Field( default=None, + min_length=1, + max_length=500, description=( "Geographic area where predator specimens were collected. " "Include site, region, and country if available. " @@ -48,17 +48,11 @@ class PredatorDietMetrics(BaseModel): examples=["Yellowstone National Park, Wyoming, USA", "Marion Island, sub-Antarctic", "Bristol, UK"], ) - study_date: Optional[ - Annotated[ - str, - constr( - min_length=4, - max_length=30, - pattern=r"^\d{4}([\-—]\d{4})?$", - ), - ] - ] = Field( + study_date: Optional[str] = Field( default=None, + min_length=4, + max_length=30, + pattern=r"^\d{4}([\-—]\d{4})?$", description=( "Year or year-range when specimens were COLLECTED (not publication year). " "Format: 'YYYY' for single year or 'YYYY-YYYY' for range (e.g., '2019' or '2019-2021'). " From 629de8fc952b1c32b09708e8e2539ff1dc94c095 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 17:54:05 -0700 Subject: [PATCH 3/9] test: consolidate duplicate tests and fix Windows /dev/null failure --- tests/test_pdf_classifier.py | 41 +++++++++++++----------------------- tests/test_pdf_extraction.py | 13 +++--------- tests/test_train_model.py | 10 +-------- 3 files changed, 19 insertions(+), 45 deletions(-) diff --git a/tests/test_pdf_classifier.py b/tests/test_pdf_classifier.py index b277197..659d6c4 100644 --- a/tests/test_pdf_classifier.py +++ b/tests/test_pdf_classifier.py @@ -42,25 +42,31 @@ def model_dir_with_mock_model(tmp_path): @patch("src.model.pdf_classifier.extract_text_from_pdf", return_value="predator stomach content analysis") def test_classify_pdf_valid_case(mock_extract, model_dir_with_mock_model, capsys): - test_pdf = Path("tests/test.pdf") - classify_pdf(test_pdf, model_dir_with_mock_model) + """Happy path: result header, prediction label, confidence percentage all present.""" + classify_pdf(Path("tests/test.pdf"), model_dir_with_mock_model) output = capsys.readouterr().out assert "PDF Classification Result" in output + assert "Prediction:" in output assert "useful" in output.lower() assert "%" in output def test_classify_pdf_missing_model(capsys, tmp_path): - """Verifier missing model directory gracefully errors.""" - model_dir = tmp_path / "empty" - model_dir.mkdir() + """Any model_dir that lacks the required artifacts should error gracefully. - classify_pdf(Path("tests/test.pdf"), model_dir) + Covers both an empty directory and a nonexistent path — both produce the + same FileNotFoundError from load_classifier(). + """ + # Empty directory + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + classify_pdf(Path("tests/test.pdf"), empty_dir) + assert "[ERROR]" in capsys.readouterr().err - output = capsys.readouterr().err - assert "[ERROR]" in output - assert "Missing model" in output + # Nonexistent directory + classify_pdf(Path("tests/test.pdf"), tmp_path / "nope") + assert "[ERROR]" in capsys.readouterr().err @patch("src.model.pdf_classifier.extract_text_from_pdf", return_value="") @@ -70,20 +76,3 @@ def test_classify_pdf_no_text(mock_extract, model_dir_with_mock_model, capsys): output = capsys.readouterr().err assert "[ERROR]" in output assert "No text extracted" in output - - -@patch("src.model.pdf_classifier.extract_text_from_pdf", return_value="diet prey stomach analysis") -def test_classify_pdf_prediction_output(mock_extract, model_dir_with_mock_model, capsys): - classify_pdf(Path("tests/valid.pdf"), model_dir_with_mock_model) - - output = capsys.readouterr().out - assert "Prediction:" in output - assert "%" in output - - -def test_classify_pdf_handles_nonexistent_dir(tmp_path, capsys): - classify_pdf(Path("tests/missing.pdf"), tmp_path / "nope") - - output = capsys.readouterr().err - assert "[ERROR]" in output - assert "Missing model" in output diff --git a/tests/test_pdf_extraction.py b/tests/test_pdf_extraction.py index d59e1e7..9ed95c2 100644 --- a/tests/test_pdf_extraction.py +++ b/tests/test_pdf_extraction.py @@ -55,21 +55,14 @@ def test_extract_text_with_ocr(tmp_path): assert text is not None -def test_save_to_file(tmp_path): - output_file = tmp_path / "output.txt" - save_to_file("hello world", output_file) - assert output_file.exists() - contents = output_file.read_text() - assert "hello" in contents - - def test_save_to_file_permission_error(monkeypatch, tmp_path): + import io + output_file = tmp_path / "readonly.txt" output_file.touch() output_file.chmod(0o400) - # Monkeypatch print to avoid cluttering stderr - monkeypatch.setattr("sys.stderr", open("/dev/null", "w")) + monkeypatch.setattr("sys.stderr", io.StringIO()) # Should not raise exception even if write fails save_to_file("data", output_file) diff --git a/tests/test_train_model.py b/tests/test_train_model.py index 6b7330a..3882669 100644 --- a/tests/test_train_model.py +++ b/tests/test_train_model.py @@ -61,6 +61,7 @@ def test_train_pdf_classifier_creates_model_and_vectorizer(sample_data, tmp_path assert result is not None assert "accuracy" in result + assert 0.0 <= result["accuracy"] <= 1.0 assert (output_dir / "pdf_classifier.json").exists() assert (output_dir / "tfidf_vectorizer.pkl").exists() @@ -92,12 +93,3 @@ def test_train_pdf_classifier_too_few_per_class(capsys): assert "Each class needs at least 2 samples" in captured.out -def test_train_pdf_classifier_output_accuracy_range(sample_data, tmp_path): - data_dir, labels_file = sample_data - texts, labels, _ = load_labeled_data(data_dir, labels_file) - - output_dir = tmp_path / "models" - result = train_pdf_classifier(texts, labels, output_dir) - - assert result is not None - assert 0.0 <= result["accuracy"] <= 1.0 From c0708b8f78ccf8aae5f871e16f037682fcd2f8ef Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 17:54:13 -0700 Subject: [PATCH 4/9] test(llm): add test coverage for models and llm_text modules --- tests/test_llm_text.py | 356 +++++++++++++++++++++++++++++++++++++++++ tests/test_models.py | 293 +++++++++++++++++++++++++++++++++ 2 files changed, 649 insertions(+) create mode 100644 tests/test_llm_text.py create mode 100644 tests/test_models.py diff --git a/tests/test_llm_text.py b/tests/test_llm_text.py new file mode 100644 index 0000000..b22c98b --- /dev/null +++ b/tests/test_llm_text.py @@ -0,0 +1,356 @@ +"""Unit tests for src/llm/llm_text.py — section extraction and text preprocessing.""" + +import sys +import tempfile +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from src.llm.llm_text import ( + _section_priority, + _score_paragraph, + _truncate_at_sentence, + classify_page, + extract_key_sections, + load_document, + split_into_pages, +) + + +# --------------------------------------------------------------------------- +# Helpers / shared fixtures +# --------------------------------------------------------------------------- + +# Realistic synthetic diet-survey text with [PAGE N] markers. +# Mimics the layout of a real paper so tests exercise real-world patterns. +_SURVEY_TEXT = """\ +[PAGE 1] +Abstract + +We examined the stomach contents of 144 Pygoscelis papua individuals +collected at Marion Island, sub-Antarctic between 1984 and 1985. +A total of 144 food samples was collected; all samples contained prey. +num_empty_stomachs = 0, num_nonempty_stomachs = 144, sample_size = 144. + +[PAGE 2] +Methods + +Specimens were collected during the 1984-1985 field season. +Stomach pumping was performed on all 144 birds; every sample yielded prey. +Study site: Marion Island, sub-Antarctic. + +[PAGE 3] +Results + +Of 144 stomachs examined, 144 were non-empty (100%). +Empty stomachs: 0. Feeding rate: 1.0. +Diet composition consisted primarily of euphausiids and fish. + +[PAGE 4] +References + +Adams NJ, Brown CR. 1989. Prey selection in the kelp gull... +Anderson DT. 1975. Embryology and phylogeny... +""" + +_SHORT_TEXT = "Short text under limit." + + +# --------------------------------------------------------------------------- +# split_into_pages +# --------------------------------------------------------------------------- + + +class TestSplitIntoPages: + def test_basic_split(self): + text = "[PAGE 1]\nPage one content.\n[PAGE 2]\nPage two content." + pages = split_into_pages(text) + assert len(pages) == 2 + assert pages[0] == (1, "\nPage one content.\n") + assert pages[1] == (2, "\nPage two content.") + + def test_preamble_before_first_marker(self): + text = "Preamble text.\n[PAGE 1]\nFirst page." + pages = split_into_pages(text) + # Preamble goes to page 0; PAGE 1 is second entry + assert pages[0] == (0, "Preamble text.\n") + assert pages[1][0] == 1 + + def test_no_page_markers_returns_empty(self): + pages = split_into_pages("No markers here.") + # Entire text is treated as preamble at page 0 + assert len(pages) == 1 + assert pages[0][0] == 0 + + def test_multi_page_document(self): + pages = split_into_pages(_SURVEY_TEXT) + page_numbers = [p for p, _ in pages] + assert 1 in page_numbers + assert 2 in page_numbers + assert 3 in page_numbers + assert 4 in page_numbers + + def test_empty_string(self): + pages = split_into_pages("") + assert pages == [] + + def test_page_content_preserved(self): + text = "[PAGE 5]\nSome content here.\n" + pages = split_into_pages(text) + assert pages[0][0] == 5 + assert "Some content here." in pages[0][1] + + +# --------------------------------------------------------------------------- +# classify_page +# --------------------------------------------------------------------------- + + +class TestClassifyPage: + def test_abstract_page_is_useful(self): + useful, priority = classify_page("Abstract\n\nWe studied Gadus morhua diets.") + assert useful is True + assert priority == 0 # first SECTION_PATTERNS entry + + def test_results_page_is_useful(self): + useful, priority = classify_page("Results\n\n144 stomachs examined.") + assert useful is True + assert priority == 1 + + def test_methods_page_is_useful(self): + useful, priority = classify_page("Methods\n\nAnimals were sampled at sea.") + assert useful is True + + def test_references_page_is_dropped(self): + useful, priority = classify_page("References\n\nSmith J. 1995. ...") + assert useful is False + assert priority == 999 + + def test_acknowledgements_page_is_dropped(self): + useful, priority = classify_page("Acknowledgements\n\nWe thank the team.") + assert useful is False + + def test_unknown_section_is_useful_with_default_priority(self): + useful, priority = classify_page("Some random section\n\nRandom text.") + assert useful is True + assert priority == len([None] * 6) # len(SECTION_PATTERNS) + + def test_bibliography_dropped(self): + useful, _ = classify_page("Bibliography\nSmith 1999...") + assert useful is False + + +# --------------------------------------------------------------------------- +# _section_priority +# --------------------------------------------------------------------------- + + +class TestSectionPriority: + def test_abstract_priority_zero(self): + assert _section_priority("Abstract") == 0 + assert _section_priority("ABSTRACT") == 0 + assert _section_priority("Summary") == 0 + + def test_results_priority_one(self): + assert _section_priority("Results") == 1 + assert _section_priority("RESULTS") == 1 + + def test_methods_priority_two(self): + assert _section_priority("Methods") == 2 + assert _section_priority("Materials and Methods") == 2 + assert _section_priority("Study Area") == 2 + + def test_introduction_priority_four(self): + assert _section_priority("Introduction") == 4 + + def test_discussion_priority_five(self): + assert _section_priority("Discussion") == 5 + assert _section_priority("Conclusions") == 5 + + def test_unknown_heading_returns_six(self): + assert _section_priority("Some Unknown Section") == 6 + assert _section_priority("Appendix A") == 6 + + def test_numbered_headings_recognized(self): + assert _section_priority("1. Abstract") == 0 + assert _section_priority("2. Methods") == 2 + assert _section_priority("3.1 Results") == 1 + + +# --------------------------------------------------------------------------- +# _score_paragraph +# --------------------------------------------------------------------------- + + +class TestScoreParagraph: + def test_empty_paragraph_scores_zero(self): + assert _score_paragraph("") == 0 + + def test_sample_size_language_scores_high(self): + para = "A total of 144 stomachs were examined." + assert _score_paragraph(para) > 0 + + def test_empty_stomach_language_scores_high(self): + para = "Of 200 individuals, 45 had empty stomachs." + assert _score_paragraph(para) > 0 + + def test_nonempty_language_scores_high(self): + para = "144 stomachs were non-empty and contained food." + assert _score_paragraph(para) > 0 + + def test_diet_keywords_score_positive(self): + para = "Stomach content analysis revealed euphausiids as the primary prey." + assert _score_paragraph(para) > 0 + + def test_date_keywords_score_positive(self): + para = "Specimens were collected during the 1984-1985 field season." + assert _score_paragraph(para) > 0 + + def test_location_keywords_score_positive(self): + para = "The study site was located on Marion Island, sub-Antarctic." + assert _score_paragraph(para) > 0 + + def test_data_rich_paragraph_scores_higher_than_plain(self): + data_para = "N=144 individuals examined; 0 empty stomachs, 144 non-empty, feeding rate 100%." + plain_para = "The weather was cold during the study." + assert _score_paragraph(data_para) > _score_paragraph(plain_para) + + def test_binomial_name_scores_nonzero(self): + para = "Pygoscelis papua was the focal species." + assert _score_paragraph(para) > 0 + + +# --------------------------------------------------------------------------- +# _truncate_at_sentence +# --------------------------------------------------------------------------- + + +class TestTruncateAtSentence: + def test_short_text_returned_unchanged(self): + text = "Short." + assert _truncate_at_sentence(text, 100) == text + + def test_truncates_at_sentence_boundary(self): + text = "First sentence. Second sentence. Third sentence." + result = _truncate_at_sentence(text, 32) + assert result.endswith(".") + assert len(result) <= 32 + + def test_falls_back_to_hard_limit(self): + text = "No sentence boundaries in this very long string of words and more words" + result = _truncate_at_sentence(text, 20) + assert len(result) == 20 + + def test_exact_limit_unchanged(self): + text = "Exactly twenty chars" + assert _truncate_at_sentence(text, len(text)) == text + + def test_respects_exclamation_and_question(self): + text = "Is this a question? Yes it is! And more text here." + result = _truncate_at_sentence(text, 25) + assert result[-1] in ("?", "!", ".") + + +# --------------------------------------------------------------------------- +# extract_key_sections +# --------------------------------------------------------------------------- + + +class TestExtractKeySections: + def test_short_text_returned_unchanged(self): + result = extract_key_sections(_SHORT_TEXT, max_chars=1000) + assert result == _SHORT_TEXT + + def test_output_within_budget(self): + budget = 500 + result = extract_key_sections(_SURVEY_TEXT, max_chars=budget) + assert len(result) <= budget + + def test_abstract_content_prioritized(self): + result = extract_key_sections(_SURVEY_TEXT, max_chars=600) + # The abstract section (PAGE 1) contains key metadata + assert "144" in result or "Pygoscelis" in result or "Marion Island" in result + + def test_references_excluded(self): + # Budget forces truncation (text is ~795 chars); references must be dropped + result = extract_key_sections(_SURVEY_TEXT, max_chars=700) + assert "Adams NJ, Brown CR. 1989." not in result + + def test_data_rich_content_included(self): + # Results page has high-scoring phrases — should be included when budget allows + result = extract_key_sections(_SURVEY_TEXT, max_chars=800) + # At least some data-bearing content should survive + assert any(kw in result for kw in ["144", "stomachs", "empty", "non-empty", "Methods", "Abstract"]) + + def test_exact_budget_limit_respected(self): + # The function pins the abstract/preamble first; budgets >= 400 are large + # enough that the pinned preamble fits and total output respects the limit. + for budget in [400, 600, 800]: + result = extract_key_sections(_SURVEY_TEXT, max_chars=budget) + assert len(result) <= budget, f"Exceeded budget={budget}: got {len(result)}" + + def test_empty_string_returns_empty(self): + result = extract_key_sections("", max_chars=500) + assert result == "" + + def test_large_budget_returns_most_content(self): + # With a very large budget the full text should come back (it's short enough) + result = extract_key_sections(_SURVEY_TEXT, max_chars=100_000) + assert result == _SURVEY_TEXT + + +# --------------------------------------------------------------------------- +# load_document +# --------------------------------------------------------------------------- + + +class TestLoadDocument: + def test_loads_txt_file(self, tmp_path): + txt_file = tmp_path / "survey.txt" + txt_file.write_text("Sample diet survey text.\n[PAGE 1]\nContent.", encoding="utf-8") + result = load_document(txt_file) + assert "Sample diet survey text." in result + assert "[PAGE 1]" in result + + def test_loads_text_extension(self, tmp_path): + text_file = tmp_path / "survey.text" + text_file.write_text("Diet content.", encoding="utf-8") + result = load_document(text_file) + assert result == "Diet content." + + def test_unsupported_extension_raises(self, tmp_path): + csv_file = tmp_path / "data.csv" + csv_file.write_text("col1,col2", encoding="utf-8") + with pytest.raises(RuntimeError, match="Unsupported file type"): + load_document(csv_file) + + def test_pdf_extension_triggers_pdf_path(self, tmp_path, monkeypatch): + """PDF loading delegates to extract_text_from_pdf — verify the call is made.""" + pdf_file = tmp_path / "paper.pdf" + pdf_file.write_bytes(b"%PDF-1.4 fake content") + + import src.llm.llm_text as llm_text_module + + monkeypatch.setattr( + "src.llm.llm_text.extract_text_from_pdf", + lambda path: f"extracted from {Path(path).name}", + raising=False, + ) + # Patch the lazy import inside load_document + import importlib + import sys as _sys + + fake_module = type(_sys)("src.preprocessing.pdf_text_extraction") + fake_module.extract_text_from_pdf = lambda path: f"extracted from {Path(path).name}" + monkeypatch.setitem(_sys.modules, "src.preprocessing.pdf_text_extraction", fake_module) + + result = load_document(pdf_file) + assert "extracted from paper.pdf" in result + + def test_encoding_error_raises_runtime(self, tmp_path): + bad_file = tmp_path / "bad.txt" + bad_file.write_bytes(b"\xff\xfe invalid utf-8 \x80\x81") + with pytest.raises(RuntimeError, match="encoding error"): + load_document(bad_file) diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..fe2c935 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,293 @@ +"""Unit tests for src/llm/models.py — PredatorDietMetrics Pydantic schema.""" + +import sys +from pathlib import Path + +import pytest +from pydantic import ValidationError + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from src.llm.models import PredatorDietMetrics + + +# --------------------------------------------------------------------------- +# Synthetic diet survey data — stands in for a real document +# --------------------------------------------------------------------------- + +# Mimics data extracted from a stomach-content study of Atlantic cod: +# - 200 individuals collected, 45 had empty stomachs, 155 had food +# - Study site: Grand Banks, Newfoundland +# - Collection period: 1998-2000 +FULL_SURVEY = dict( + species_name="Gadus morhua", + study_location="Grand Banks, Newfoundland, Canada", + study_date="1998-2000", + num_empty_stomachs=45, + num_nonempty_stomachs=155, + sample_size=200, +) + +# All-food stomach-pumping study (num_empty always 0) +PUMP_SURVEY = dict( + species_name="Pygoscelis papua", + study_location="Marion Island, sub-Antarctic", + study_date="1984-1985", + num_empty_stomachs=0, + num_nonempty_stomachs=144, + sample_size=144, +) + +# Minimal survey — only species and sample size known +MINIMAL_SURVEY = dict( + species_name="Ursus arctos", + study_location=None, + study_date="2020", + num_empty_stomachs=None, + num_nonempty_stomachs=None, + sample_size=23, +) + + +# --------------------------------------------------------------------------- +# Basic construction +# --------------------------------------------------------------------------- + + +class TestBasicConstruction: + def test_full_survey_constructs(self): + m = PredatorDietMetrics(**FULL_SURVEY) + assert m.species_name == "Gadus morhua" + assert m.study_location == "Grand Banks, Newfoundland, Canada" + assert m.study_date == "1998-2000" + assert m.num_empty_stomachs == 45 + assert m.num_nonempty_stomachs == 155 + assert m.sample_size == 200 + + def test_all_null_optional_fields(self): + m = PredatorDietMetrics() + assert m.species_name is None + assert m.study_location is None + assert m.study_date is None + assert m.num_empty_stomachs is None + assert m.num_nonempty_stomachs is None + assert m.sample_size is None + + def test_minimal_survey_constructs(self): + m = PredatorDietMetrics(**MINIMAL_SURVEY) + assert m.species_name == "Ursus arctos" + assert m.sample_size == 23 + assert m.num_empty_stomachs is None + assert m.num_nonempty_stomachs is None + + +# --------------------------------------------------------------------------- +# sample_size auto-reconciliation +# --------------------------------------------------------------------------- + + +class TestSampleSizeReconciliation: + def test_sample_size_set_from_empty_plus_nonempty(self): + m = PredatorDietMetrics(num_empty_stomachs=45, num_nonempty_stomachs=155, sample_size=None) + assert m.sample_size == 200 + + def test_wrong_sample_size_corrected(self): + """Model validator should override an incorrect sample_size.""" + m = PredatorDietMetrics(num_empty_stomachs=10, num_nonempty_stomachs=90, sample_size=999) + assert m.sample_size == 100 + + def test_correct_sample_size_unchanged(self): + m = PredatorDietMetrics(**FULL_SURVEY) + assert m.sample_size == 200 + + def test_pump_survey_zero_empty(self): + m = PredatorDietMetrics(**PUMP_SURVEY) + assert m.num_empty_stomachs == 0 + assert m.num_nonempty_stomachs == 144 + assert m.sample_size == 144 + + def test_only_empty_provided_no_reconciliation(self): + """Cannot reconcile without both counts — sample_size stays None.""" + m = PredatorDietMetrics(num_empty_stomachs=10, num_nonempty_stomachs=None, sample_size=None) + assert m.sample_size is None + + def test_only_nonempty_provided_no_reconciliation(self): + m = PredatorDietMetrics(num_empty_stomachs=None, num_nonempty_stomachs=90, sample_size=None) + assert m.sample_size is None + + +# --------------------------------------------------------------------------- +# fraction_feeding computed field +# --------------------------------------------------------------------------- + + +class TestFractionFeeding: + def test_fraction_computed_correctly(self): + m = PredatorDietMetrics(**FULL_SURVEY) + assert m.fraction_feeding == pytest.approx(0.775, abs=1e-4) + + def test_fraction_all_feeding(self): + m = PredatorDietMetrics(**PUMP_SURVEY) + assert m.fraction_feeding == pytest.approx(1.0) + + def test_fraction_none_when_nonempty_missing(self): + m = PredatorDietMetrics(num_empty_stomachs=5, num_nonempty_stomachs=None, sample_size=50) + assert m.fraction_feeding is None + + def test_fraction_none_when_sample_size_missing(self): + m = PredatorDietMetrics(num_nonempty_stomachs=30, sample_size=None) + assert m.fraction_feeding is None + + def test_fraction_none_when_all_null(self): + m = PredatorDietMetrics() + assert m.fraction_feeding is None + + def test_fraction_rounded_to_4_decimals(self): + # 2 empty + 1 nonempty → sample_size=3, fraction = 1/3 = 0.3333 + m = PredatorDietMetrics(num_empty_stomachs=2, num_nonempty_stomachs=1) + assert m.fraction_feeding == pytest.approx(0.3333, abs=1e-4) + + def test_fraction_not_in_model_dump_base_fields(self): + """fraction_feeding is a computed field and should appear in model_dump().""" + m = PredatorDietMetrics(**FULL_SURVEY) + d = m.model_dump() + assert "fraction_feeding" in d + + +# --------------------------------------------------------------------------- +# Field-level validation — species_name +# --------------------------------------------------------------------------- + + +class TestSpeciesNameValidation: + def test_valid_binomial(self): + m = PredatorDietMetrics(species_name="Canis lupus") + assert m.species_name == "Canis lupus" + + def test_valid_three_word_name(self): + m = PredatorDietMetrics(species_name="Canis lupus familiaris") + assert m.species_name == "Canis lupus familiaris" + + def test_null_species_allowed(self): + m = PredatorDietMetrics(species_name=None) + assert m.species_name is None + + def test_lowercase_genus_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(species_name="canis lupus") + + def test_all_caps_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(species_name="CANIS LUPUS") + + def test_too_short_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(species_name="Ab") + + def test_whitespace_stripped(self): + m = PredatorDietMetrics(species_name=" Vulpes vulpes ") + assert m.species_name == "Vulpes vulpes" + + +# --------------------------------------------------------------------------- +# Field-level validation — study_date +# --------------------------------------------------------------------------- + + +class TestStudyDateValidation: + def test_single_year(self): + m = PredatorDietMetrics(study_date="2019") + assert m.study_date == "2019" + + def test_year_range_hyphen(self): + m = PredatorDietMetrics(study_date="2015-2018") + assert m.study_date == "2015-2018" + + def test_null_date_allowed(self): + m = PredatorDietMetrics(study_date=None) + assert m.study_date is None + + def test_text_date_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(study_date="March 2019") + + def test_partial_year_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(study_date="19") + + def test_three_year_range_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(study_date="2015-2018-2020") + + +# --------------------------------------------------------------------------- +# Field-level validation — count fields +# --------------------------------------------------------------------------- + + +class TestCountFieldValidation: + def test_negative_empty_stomachs_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(num_empty_stomachs=-1) + + def test_negative_nonempty_stomachs_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(num_nonempty_stomachs=-5) + + def test_zero_empty_stomachs_allowed(self): + m = PredatorDietMetrics(num_empty_stomachs=0) + assert m.num_empty_stomachs == 0 + + def test_zero_sample_size_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(sample_size=0) + + def test_negative_sample_size_rejected(self): + with pytest.raises(ValidationError): + PredatorDietMetrics(sample_size=-10) + + def test_large_counts_accepted(self): + m = PredatorDietMetrics(num_empty_stomachs=0, num_nonempty_stomachs=10000, sample_size=10000) + assert m.sample_size == 10000 + + +# --------------------------------------------------------------------------- +# Serialization round-trip +# --------------------------------------------------------------------------- + + +class TestSerialization: + def test_model_dump_round_trip(self): + m = PredatorDietMetrics(**FULL_SURVEY) + d = m.model_dump() + m2 = PredatorDietMetrics.model_validate(d) + assert m2.species_name == m.species_name + assert m2.sample_size == m.sample_size + assert m2.fraction_feeding == m.fraction_feeding + + def test_model_validate_json(self): + import json + + payload = json.dumps( + { + "species_name": "Vulpes vulpes", + "study_location": "Bristol, UK", + "study_date": "2015-2018", + "num_empty_stomachs": 12, + "num_nonempty_stomachs": 88, + "sample_size": 100, + } + ) + m = PredatorDietMetrics.model_validate_json(payload) + assert m.species_name == "Vulpes vulpes" + assert m.fraction_feeding == pytest.approx(0.88) + + def test_model_json_schema_contains_required_fields(self): + schema = PredatorDietMetrics.model_json_schema() + props = schema.get("properties", {}) + assert "species_name" in props + assert "study_location" in props + assert "study_date" in props + assert "num_empty_stomachs" in props + assert "num_nonempty_stomachs" in props + assert "sample_size" in props From e7fa43d28408e62adbc26013bed92623bf1f5fdb Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 18:04:53 -0700 Subject: [PATCH 5/9] chore: fix black and flake8 violations across src/ and tests/ --- src/llm/llm_text.py | 4 +- src/model/train_model.py | 1 - src/preprocessing/pdf_text_extraction.py | 2 +- src/preprocessing/test.py | 9 + tests/local_llm_debug.py | 209 ++++++++++++++++++++++ tests/test_extraction.py | 218 +++++++++++++++++++++++ tests/test_train_model.py | 2 - 7 files changed, 439 insertions(+), 6 deletions(-) create mode 100644 src/preprocessing/test.py create mode 100644 tests/local_llm_debug.py create mode 100644 tests/test_extraction.py diff --git a/src/llm/llm_text.py b/src/llm/llm_text.py index 2083115..9bc8862 100644 --- a/src/llm/llm_text.py +++ b/src/llm/llm_text.py @@ -411,12 +411,12 @@ def load_document(file_path: Path) -> str: suffix = file_path.suffix.lower() if suffix == '.pdf': - print(f"[INFO] Reading PDF file...", file=sys.stderr) + print("[INFO] Reading PDF file...", file=sys.stderr) from src.preprocessing.pdf_text_extraction import extract_text_from_pdf return extract_text_from_pdf(str(file_path)) elif suffix in ['.txt', '.text']: - print(f"[INFO] Reading text file...", file=sys.stderr) + print("[INFO] Reading text file...", file=sys.stderr) try: with open(file_path, "r", encoding="utf-8") as f: return f.read() diff --git a/src/model/train_model.py b/src/model/train_model.py index d9663e7..a557e4f 100644 --- a/src/model/train_model.py +++ b/src/model/train_model.py @@ -10,7 +10,6 @@ import sys from pathlib import Path from collections import Counter -from pathlib import Path import joblib import json diff --git a/src/preprocessing/pdf_text_extraction.py b/src/preprocessing/pdf_text_extraction.py index b7fa00c..80beadf 100644 --- a/src/preprocessing/pdf_text_extraction.py +++ b/src/preprocessing/pdf_text_extraction.py @@ -227,7 +227,7 @@ def extract_text_from_pdf_bytes(data: bytes, skip_ocr: bool = False) -> str: return "" if not skip_ocr and check_spelling(text) > MAX_SPELLING_ERROR_RATE: - print(f"[INFO] High misspelling rate — falling back to OCR", file=sys.stderr) + print("[INFO] High misspelling rate — falling back to OCR", file=sys.stderr) log.warning("High spelling error rate in in-memory PDF — falling back to OCR extraction.") text = [] try: diff --git a/src/preprocessing/test.py b/src/preprocessing/test.py new file mode 100644 index 0000000..c63a494 --- /dev/null +++ b/src/preprocessing/test.py @@ -0,0 +1,9 @@ +import fitz + +with fitz.open("data/useful/Adams_1989.pdf") as doc: + for page_num, page in enumerate(doc, start=1): + tabs = page.find_tables() + print(f"Page {page_num}: {len(tabs.tables)} tables found") + if tabs.tables: + for i, tab in enumerate(tabs.tables): + print(f" Table {i + 1}: {tab.bbox}") diff --git a/tests/local_llm_debug.py b/tests/local_llm_debug.py new file mode 100644 index 0000000..3d4a558 --- /dev/null +++ b/tests/local_llm_debug.py @@ -0,0 +1,209 @@ +"""Debug wrapper for local_llm.py - saves all intermediate outputs. + +This patches the extraction functions to save debug output without modifying local_llm.py +""" + +import sys +import json +from pathlib import Path + +debug_dir = None +call_counter = {"metadata": 0, "stomach": 0} + + +def save_debug(filename, content, description=""): + """Save debug content to file.""" + if debug_dir: + filepath = debug_dir / filename + with open(filepath, 'w', encoding='utf-8') as f: + if description: + f.write(f"# {description}\n") + f.write(f"# {'=' * 60}\n\n") + f.write(content) + print(f"[DEBUG] Saved: {filepath}", file=sys.__stderr__) + + +# Monkey-patch the chat function to intercept prompts and responses +from ollama import chat as original_chat + + +def patched_chat(messages, model, format=None): + """Intercept chat calls to save prompts and responses.""" + prompt = messages[0]['content'] if messages else "" + + # Determine which pass this is + if "species_name" in str(format): + pass_type = "metadata" + call_counter["metadata"] += 1 + call_num = call_counter["metadata"] + + # Save prompt + save_debug(f"02_metadata_prompt_{call_num}.txt", prompt, f"Metadata extraction prompt (call {call_num})") + elif "num_empty_stomachs" in str(format): + pass_type = "stomach" + call_counter["stomach"] += 1 + call_num = call_counter["stomach"] + + # Save prompt + save_debug(f"05_stomach_data_prompt_{call_num}.txt", prompt, f"Stomach data extraction prompt (call {call_num})") + else: + pass_type = "unknown" + call_num = 0 + + # Call original + response = original_chat(messages, model, format) + + # Save response + if pass_type == "metadata": + save_debug(f"03_metadata_response_{call_num}.json", response.message.content, f"Metadata extraction response (call {call_num})") + elif pass_type == "stomach": + save_debug(f"06_stomach_data_response_{call_num}.json", response.message.content, f"Stomach data extraction response (call {call_num})") + + return response + + +# Apply patches +import ollama + +ollama.chat = patched_chat + + +def main(): + import argparse + + parser = argparse.ArgumentParser(description="Debug wrapper for local_llm.py") + parser.add_argument("pdf", type=str, help="Path to PDF") + parser.add_argument("--model", type=str, default="llama3.1:8b") + parser.add_argument("--output-dir", type=str, default="data/results") + parser.add_argument("--debug-dir", type=str, default="debug_output", help="Where to save debug files") + + args = parser.parse_args() + + # Setup debug directory + global debug_dir + pdf_name = Path(args.pdf).stem + debug_dir = Path(args.debug_dir) / pdf_name + debug_dir.mkdir(parents=True, exist_ok=True) + + print(f"[DEBUG] Debug output will be saved to: {debug_dir}", file=sys.__stderr__) + + # Add src directory to path + # Script is in tests/local_llm_debug.py + # Get project root (parent of tests/) then add src/ + project_root = Path(__file__).resolve().parent.parent + src_path = project_root / "src" + + if not src_path.exists(): + print(f"[ERROR] src directory not found at: {src_path}", file=sys.__stderr__) + sys.exit(1) + + if str(src_path) not in sys.path: + sys.path.insert(0, str(src_path)) + + print(f"[DEBUG] Added to path: {src_path}", file=sys.__stderr__) + + # Import the module correctly + from llm import llm_client + + # Store original functions + original_extract_metadata = llm_client.extract_metadata_with_search + original_extract_stomach = llm_client.extract_stomach_data_with_search + original_extract_counts = llm_client.extract_stomach_counts_from_text + + # Patch extract_metadata_with_search + def patched_extract_metadata(text, model): + # Save the context being analyzed + context = text[:12000] + save_debug("01_metadata_context.txt", context, "Context sent to LLM for metadata extraction (first 12000 chars)") + + # Call original + result = original_extract_metadata(text, model) + + # Save result + save_debug("03_metadata_result.json", json.dumps(result, indent=2), "Final metadata extraction result") + + return result + + # Patch extract_stomach_data_with_search + def patched_extract_stomach(text, model): + # Save full text first + save_debug("00_full_text.txt", text, f"Full extracted text ({len(text)} characters)") + + # Call regex extraction + regex_result = original_extract_counts(text) + save_debug("04_regex_extraction.json", json.dumps(regex_result, indent=2), "Regex-based stomach count extraction (ground truth)") + + # Find stomach-related passages + import re + + stomach_pattern = r'.{0,800}(?:stomachs?|empty|sample\s+size|n\s*=).{0,800}' + matches = re.findall(stomach_pattern, text, re.IGNORECASE) + context = '\n\n---\n\n'.join(matches[:15]) + + if not context: + context = text[:15000] + + save_debug("04_stomach_data_context.txt", context, "Context sent to LLM for stomach data extraction") + + # Call original (which will call our patched chat function) + result = original_extract_stomach(text, model) + + # Save final result + save_debug("06_stomach_data_result.json", json.dumps(result, indent=2), "Final stomach data extraction result") + + return result + + # Patch extract_stomach_counts_from_text to log what it finds + def patched_extract_counts(text): + result = original_extract_counts(text) + + # Save what regex patterns found + import re + + debug_info = {"result": result, "patterns_found": []} + + # Check what each pattern matched + empty_pattern = r'(\d+)\s+stomachs?\s+(?:were\s+)?(?:found\s+)?empty' + empty_matches = re.findall(empty_pattern, text, re.IGNORECASE) + if empty_matches: + debug_info["patterns_found"].append({"pattern": "empty stomachs", "matches": empty_matches}) + + total_pattern = r'(?:total\s+of\s+|^|\.\s+)(\d+)\s+stomachs' + total_matches = re.findall(total_pattern, text, re.IGNORECASE | re.MULTILINE) + if total_matches: + debug_info["patterns_found"].append({"pattern": "total stomachs", "matches": total_matches}) + + save_debug("04_regex_patterns_detail.json", json.dumps(debug_info, indent=2), "Detailed regex pattern matching results") + + return result + + # Apply all patches + llm_client.extract_metadata_with_search = patched_extract_metadata + llm_client.extract_stomach_data_with_search = patched_extract_stomach + llm_client.extract_stomach_counts_from_text = patched_extract_counts + + # Set up args for local_llm + sys.argv = ["local_llm.py", args.pdf, "--model", args.model, "--output-dir", args.output_dir] + + # Run it + try: + llm_client.main() + except SystemExit: + pass + + print(f"\n{'=' * 60}", file=sys.__stderr__) + print(f"[DEBUG] All debug files saved to: {debug_dir}", file=sys.__stderr__) + print(f"{'=' * 60}\n", file=sys.__stderr__) + print("[DEBUG] Review these files to diagnose issues:", file=sys.__stderr__) + print(f" 1. {debug_dir}/00_full_text.txt - Full extracted PDF text", file=sys.__stderr__) + print(f" 2. {debug_dir}/01_metadata_context.txt - Context for metadata extraction", file=sys.__stderr__) + print(f" 3. {debug_dir}/02_metadata_prompt_1.txt - Prompt sent to LLM", file=sys.__stderr__) + print(f" 4. {debug_dir}/03_metadata_response_1.json - LLM response", file=sys.__stderr__) + print(f" 5. {debug_dir}/04_regex_extraction.json - What regex found", file=sys.__stderr__) + print(f" 6. {debug_dir}/04_stomach_data_context.txt - Context for stomach data", file=sys.__stderr__) + print(f" 7. {debug_dir}/05_stomach_data_prompt_1.txt - Prompt sent to LLM", file=sys.__stderr__) + print(f" 8. {debug_dir}/06_stomach_data_response_1.json - LLM response\n", file=sys.__stderr__) + + +if __name__ == "__main__": + main() diff --git a/tests/test_extraction.py b/tests/test_extraction.py new file mode 100644 index 0000000..bd1e26c --- /dev/null +++ b/tests/test_extraction.py @@ -0,0 +1,218 @@ +"""Test script to compare original vs improved LLM extraction. + +Usage: + python test_comparison.py path/to/text_file.txt path/to/original.pdf + +This script runs both the original and improved extraction methods and +compares the results side-by-side. +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import Dict + + +def load_json_results(filepath: str) -> Dict: + """Load JSON results from file.""" + try: + with open(filepath, 'r', encoding='utf-8') as f: + return json.load(f) + except Exception as e: + print(f"[ERROR] Could not load {filepath}: {e}") + return {} + + +def count_null_fields(metrics: Dict) -> int: + """Count how many fields are null.""" + null_count = 0 + fields_to_check = ['species_name', 'study_location', 'study_date', 'num_empty_stomachs', 'num_nonempty_stomachs', 'sample_size'] + for field in fields_to_check: + if metrics.get(field) is None: + null_count += 1 + return null_count + + +def print_comparison_table(original_metrics: Dict, improved_metrics: Dict): + """Print a side-by-side comparison of extraction results.""" + + print("\n" + "=" * 80) + print("EXTRACTION COMPARISON: Original vs Improved") + print("=" * 80) + + fields = [ + ('species_name', 'Species Name'), + ('study_location', 'Study Location'), + ('study_date', 'Study Date'), + ('num_empty_stomachs', 'Empty Stomachs'), + ('num_nonempty_stomachs', 'Non-Empty Stomachs'), + ('sample_size', 'Sample Size'), + ('fraction_feeding', 'Fraction Feeding'), + ] + + print(f"\n{'Field':<25} {'Original':<30} {'Improved':<30}") + print("-" * 80) + + for field_key, field_name in fields: + orig_val = original_metrics.get(field_key) + impr_val = improved_metrics.get(field_key) + + orig_str = str(orig_val) if orig_val is not None else "NULL" + impr_str = str(impr_val) if impr_val is not None else "NULL" + + # Highlight improvements + marker = "" + if orig_val is None and impr_val is not None: + marker = " ✓ IMPROVED" + elif orig_val != impr_val and impr_val is not None: + marker = " ✓ CHANGED" + + print(f"{field_name:<25} {orig_str:<30} {impr_str:<30}{marker}") + + print("-" * 80) + + # Summary statistics + orig_null = count_null_fields(original_metrics) + impr_null = count_null_fields(improved_metrics) + + print(f"\nNull Fields: {orig_null}/6 {impr_null}/6") + print(f"Completeness: {((6 - orig_null) / 6) * 100:.1f}% {((6 - impr_null) / 6) * 100:.1f}%") + + if impr_null < orig_null: + print(f"\n✓ IMPROVEMENT: {orig_null - impr_null} fewer null fields!") + elif impr_null > orig_null: + print(f"\n✗ REGRESSION: {impr_null - orig_null} more null fields") + else: + print("\n→ No change in null field count") + + print("=" * 80 + "\n") + + +def main(): + parser = argparse.ArgumentParser(description="Compare original vs improved LLM extraction") + parser.add_argument("text_file", type=str, help="Path to preprocessed text file") + parser.add_argument("pdf_file", type=str, help="Path to original PDF file") + parser.add_argument("--model", type=str, default="llama3.1:8b", help="Model to use for both extractions") + parser.add_argument("--output-dir", type=str, default="data/test_comparison", help="Output directory for comparison results") + + args = parser.parse_args() + + text_path = Path(args.text_file) + pdf_path = Path(args.pdf_file) + + if not text_path.exists(): + print(f"[ERROR] Text file not found: {text_path}") + sys.exit(1) + + if not pdf_path.exists(): + print(f"[ERROR] PDF file not found: {pdf_path}") + sys.exit(1) + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + print("\n" + "=" * 80) + print("RUNNING EXTRACTION COMPARISON TEST") + print("=" * 80) + print(f"Text file: {text_path.name}") + print(f"PDF file: {pdf_path.name}") + print(f"Model: {args.model}") + print("=" * 80 + "\n") + + # Run original extraction + print("[1/2] Running ORIGINAL extraction...") + print("-" * 80) + + import subprocess + + original_output = output_dir / f"{text_path.stem}_original_results.json" + + try: + result = subprocess.run([sys.executable, "local_llm.py", str(text_path), "--model", args.model, "--output-dir", str(output_dir)], capture_output=True, text=True, timeout=300) + + if result.returncode != 0: + print("[ERROR] Original extraction failed:") + print(result.stderr) + sys.exit(1) + + # Rename output to distinguish it + default_output = output_dir / f"{text_path.stem}_results.json" + if default_output.exists(): + default_output.rename(original_output) + + except subprocess.TimeoutExpired: + print("[ERROR] Original extraction timed out") + sys.exit(1) + except Exception as e: + print(f"[ERROR] Original extraction failed: {e}") + sys.exit(1) + + print("✓ Original extraction complete\n") + + # Run improved extraction + print("[2/2] Running IMPROVED extraction...") + print("-" * 80) + + improved_output = output_dir / f"{text_path.stem}_improved_results.json" + + try: + result = subprocess.run( + [sys.executable, "local_llm_improved.py", str(text_path), "--pdf", str(pdf_path), "--model", args.model, "--output-dir", str(output_dir)], capture_output=True, text=True, timeout=300 + ) + + if result.returncode != 0: + print("[ERROR] Improved extraction failed:") + print(result.stderr) + sys.exit(1) + + # Rename output to distinguish it + default_output = output_dir / f"{text_path.stem}_results.json" + if default_output.exists(): + default_output.rename(improved_output) + + except subprocess.TimeoutExpired: + print("[ERROR] Improved extraction timed out") + sys.exit(1) + except Exception as e: + print(f"[ERROR] Improved extraction failed: {e}") + sys.exit(1) + + print("✓ Improved extraction complete\n") + + # Load and compare results + original_data = load_json_results(str(original_output)) + improved_data = load_json_results(str(improved_output)) + + if not original_data or not improved_data: + print("[ERROR] Could not load results for comparison") + sys.exit(1) + + original_metrics = original_data.get('metrics', {}) + improved_metrics = improved_data.get('metrics', {}) + + # Print comparison + print_comparison_table(original_metrics, improved_metrics) + + # Save comparison report + comparison_report = { + 'text_file': text_path.name, + 'pdf_file': pdf_path.name, + 'model': args.model, + 'original': {'metrics': original_metrics, 'null_count': count_null_fields(original_metrics), 'completeness': f"{((6 - count_null_fields(original_metrics)) / 6) * 100:.1f}%"}, + 'improved': {'metrics': improved_metrics, 'null_count': count_null_fields(improved_metrics), 'completeness': f"{((6 - count_null_fields(improved_metrics)) / 6) * 100:.1f}%"}, + 'improvement': { + 'fields_gained': count_null_fields(original_metrics) - count_null_fields(improved_metrics), + 'is_better': count_null_fields(improved_metrics) < count_null_fields(original_metrics), + }, + } + + report_path = output_dir / f"{text_path.stem}_comparison_report.json" + with open(report_path, 'w', encoding='utf-8') as f: + json.dump(comparison_report, f, indent=2) + + print(f"Detailed comparison report saved to: {report_path}\n") + + +if __name__ == "__main__": + main() diff --git a/tests/test_train_model.py b/tests/test_train_model.py index 3882669..37d8418 100644 --- a/tests/test_train_model.py +++ b/tests/test_train_model.py @@ -91,5 +91,3 @@ def test_train_pdf_classifier_too_few_per_class(capsys): captured = capsys.readouterr() assert result is None assert "Each class needs at least 2 samples" in captured.out - - From 4003a682a3ed7ee3be1ebb5172709cdc6c837879 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 18:09:34 -0700 Subject: [PATCH 6/9] fix(llm): restore missing retry info log line in llm_client --- src/llm/llm_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llm/llm_client.py b/src/llm/llm_client.py index dcbea52..cd25900 100644 --- a/src/llm/llm_client.py +++ b/src/llm/llm_client.py @@ -147,6 +147,7 @@ def extract_metrics_from_text( if not _retry and missing: print( + f" [INFO] Retry: {', '.join(missing)} came back null — re-prompting", file=sys.stderr, ) From b1a37cbc26d8d37392ca8f57fff62438fb2a4d0c Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 18:14:20 -0700 Subject: [PATCH 7/9] chore: remove local_llm_debug.py from repo --- tests/local_llm_debug.py | 209 --------------------------------------- 1 file changed, 209 deletions(-) delete mode 100644 tests/local_llm_debug.py diff --git a/tests/local_llm_debug.py b/tests/local_llm_debug.py deleted file mode 100644 index 3d4a558..0000000 --- a/tests/local_llm_debug.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Debug wrapper for local_llm.py - saves all intermediate outputs. - -This patches the extraction functions to save debug output without modifying local_llm.py -""" - -import sys -import json -from pathlib import Path - -debug_dir = None -call_counter = {"metadata": 0, "stomach": 0} - - -def save_debug(filename, content, description=""): - """Save debug content to file.""" - if debug_dir: - filepath = debug_dir / filename - with open(filepath, 'w', encoding='utf-8') as f: - if description: - f.write(f"# {description}\n") - f.write(f"# {'=' * 60}\n\n") - f.write(content) - print(f"[DEBUG] Saved: {filepath}", file=sys.__stderr__) - - -# Monkey-patch the chat function to intercept prompts and responses -from ollama import chat as original_chat - - -def patched_chat(messages, model, format=None): - """Intercept chat calls to save prompts and responses.""" - prompt = messages[0]['content'] if messages else "" - - # Determine which pass this is - if "species_name" in str(format): - pass_type = "metadata" - call_counter["metadata"] += 1 - call_num = call_counter["metadata"] - - # Save prompt - save_debug(f"02_metadata_prompt_{call_num}.txt", prompt, f"Metadata extraction prompt (call {call_num})") - elif "num_empty_stomachs" in str(format): - pass_type = "stomach" - call_counter["stomach"] += 1 - call_num = call_counter["stomach"] - - # Save prompt - save_debug(f"05_stomach_data_prompt_{call_num}.txt", prompt, f"Stomach data extraction prompt (call {call_num})") - else: - pass_type = "unknown" - call_num = 0 - - # Call original - response = original_chat(messages, model, format) - - # Save response - if pass_type == "metadata": - save_debug(f"03_metadata_response_{call_num}.json", response.message.content, f"Metadata extraction response (call {call_num})") - elif pass_type == "stomach": - save_debug(f"06_stomach_data_response_{call_num}.json", response.message.content, f"Stomach data extraction response (call {call_num})") - - return response - - -# Apply patches -import ollama - -ollama.chat = patched_chat - - -def main(): - import argparse - - parser = argparse.ArgumentParser(description="Debug wrapper for local_llm.py") - parser.add_argument("pdf", type=str, help="Path to PDF") - parser.add_argument("--model", type=str, default="llama3.1:8b") - parser.add_argument("--output-dir", type=str, default="data/results") - parser.add_argument("--debug-dir", type=str, default="debug_output", help="Where to save debug files") - - args = parser.parse_args() - - # Setup debug directory - global debug_dir - pdf_name = Path(args.pdf).stem - debug_dir = Path(args.debug_dir) / pdf_name - debug_dir.mkdir(parents=True, exist_ok=True) - - print(f"[DEBUG] Debug output will be saved to: {debug_dir}", file=sys.__stderr__) - - # Add src directory to path - # Script is in tests/local_llm_debug.py - # Get project root (parent of tests/) then add src/ - project_root = Path(__file__).resolve().parent.parent - src_path = project_root / "src" - - if not src_path.exists(): - print(f"[ERROR] src directory not found at: {src_path}", file=sys.__stderr__) - sys.exit(1) - - if str(src_path) not in sys.path: - sys.path.insert(0, str(src_path)) - - print(f"[DEBUG] Added to path: {src_path}", file=sys.__stderr__) - - # Import the module correctly - from llm import llm_client - - # Store original functions - original_extract_metadata = llm_client.extract_metadata_with_search - original_extract_stomach = llm_client.extract_stomach_data_with_search - original_extract_counts = llm_client.extract_stomach_counts_from_text - - # Patch extract_metadata_with_search - def patched_extract_metadata(text, model): - # Save the context being analyzed - context = text[:12000] - save_debug("01_metadata_context.txt", context, "Context sent to LLM for metadata extraction (first 12000 chars)") - - # Call original - result = original_extract_metadata(text, model) - - # Save result - save_debug("03_metadata_result.json", json.dumps(result, indent=2), "Final metadata extraction result") - - return result - - # Patch extract_stomach_data_with_search - def patched_extract_stomach(text, model): - # Save full text first - save_debug("00_full_text.txt", text, f"Full extracted text ({len(text)} characters)") - - # Call regex extraction - regex_result = original_extract_counts(text) - save_debug("04_regex_extraction.json", json.dumps(regex_result, indent=2), "Regex-based stomach count extraction (ground truth)") - - # Find stomach-related passages - import re - - stomach_pattern = r'.{0,800}(?:stomachs?|empty|sample\s+size|n\s*=).{0,800}' - matches = re.findall(stomach_pattern, text, re.IGNORECASE) - context = '\n\n---\n\n'.join(matches[:15]) - - if not context: - context = text[:15000] - - save_debug("04_stomach_data_context.txt", context, "Context sent to LLM for stomach data extraction") - - # Call original (which will call our patched chat function) - result = original_extract_stomach(text, model) - - # Save final result - save_debug("06_stomach_data_result.json", json.dumps(result, indent=2), "Final stomach data extraction result") - - return result - - # Patch extract_stomach_counts_from_text to log what it finds - def patched_extract_counts(text): - result = original_extract_counts(text) - - # Save what regex patterns found - import re - - debug_info = {"result": result, "patterns_found": []} - - # Check what each pattern matched - empty_pattern = r'(\d+)\s+stomachs?\s+(?:were\s+)?(?:found\s+)?empty' - empty_matches = re.findall(empty_pattern, text, re.IGNORECASE) - if empty_matches: - debug_info["patterns_found"].append({"pattern": "empty stomachs", "matches": empty_matches}) - - total_pattern = r'(?:total\s+of\s+|^|\.\s+)(\d+)\s+stomachs' - total_matches = re.findall(total_pattern, text, re.IGNORECASE | re.MULTILINE) - if total_matches: - debug_info["patterns_found"].append({"pattern": "total stomachs", "matches": total_matches}) - - save_debug("04_regex_patterns_detail.json", json.dumps(debug_info, indent=2), "Detailed regex pattern matching results") - - return result - - # Apply all patches - llm_client.extract_metadata_with_search = patched_extract_metadata - llm_client.extract_stomach_data_with_search = patched_extract_stomach - llm_client.extract_stomach_counts_from_text = patched_extract_counts - - # Set up args for local_llm - sys.argv = ["local_llm.py", args.pdf, "--model", args.model, "--output-dir", args.output_dir] - - # Run it - try: - llm_client.main() - except SystemExit: - pass - - print(f"\n{'=' * 60}", file=sys.__stderr__) - print(f"[DEBUG] All debug files saved to: {debug_dir}", file=sys.__stderr__) - print(f"{'=' * 60}\n", file=sys.__stderr__) - print("[DEBUG] Review these files to diagnose issues:", file=sys.__stderr__) - print(f" 1. {debug_dir}/00_full_text.txt - Full extracted PDF text", file=sys.__stderr__) - print(f" 2. {debug_dir}/01_metadata_context.txt - Context for metadata extraction", file=sys.__stderr__) - print(f" 3. {debug_dir}/02_metadata_prompt_1.txt - Prompt sent to LLM", file=sys.__stderr__) - print(f" 4. {debug_dir}/03_metadata_response_1.json - LLM response", file=sys.__stderr__) - print(f" 5. {debug_dir}/04_regex_extraction.json - What regex found", file=sys.__stderr__) - print(f" 6. {debug_dir}/04_stomach_data_context.txt - Context for stomach data", file=sys.__stderr__) - print(f" 7. {debug_dir}/05_stomach_data_prompt_1.txt - Prompt sent to LLM", file=sys.__stderr__) - print(f" 8. {debug_dir}/06_stomach_data_response_1.json - LLM response\n", file=sys.__stderr__) - - -if __name__ == "__main__": - main() From b2b244d25d011e2b4cfc0bd5c69c0de27ce36b22 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 18:16:40 -0700 Subject: [PATCH 8/9] chore: remove stray debug and comparison scripts --- src/preprocessing/test.py | 9 -- tests/test_extraction.py | 218 -------------------------------------- 2 files changed, 227 deletions(-) delete mode 100644 src/preprocessing/test.py delete mode 100644 tests/test_extraction.py diff --git a/src/preprocessing/test.py b/src/preprocessing/test.py deleted file mode 100644 index c63a494..0000000 --- a/src/preprocessing/test.py +++ /dev/null @@ -1,9 +0,0 @@ -import fitz - -with fitz.open("data/useful/Adams_1989.pdf") as doc: - for page_num, page in enumerate(doc, start=1): - tabs = page.find_tables() - print(f"Page {page_num}: {len(tabs.tables)} tables found") - if tabs.tables: - for i, tab in enumerate(tabs.tables): - print(f" Table {i + 1}: {tab.bbox}") diff --git a/tests/test_extraction.py b/tests/test_extraction.py deleted file mode 100644 index bd1e26c..0000000 --- a/tests/test_extraction.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Test script to compare original vs improved LLM extraction. - -Usage: - python test_comparison.py path/to/text_file.txt path/to/original.pdf - -This script runs both the original and improved extraction methods and -compares the results side-by-side. -""" - -import argparse -import json -import sys -from pathlib import Path -from typing import Dict - - -def load_json_results(filepath: str) -> Dict: - """Load JSON results from file.""" - try: - with open(filepath, 'r', encoding='utf-8') as f: - return json.load(f) - except Exception as e: - print(f"[ERROR] Could not load {filepath}: {e}") - return {} - - -def count_null_fields(metrics: Dict) -> int: - """Count how many fields are null.""" - null_count = 0 - fields_to_check = ['species_name', 'study_location', 'study_date', 'num_empty_stomachs', 'num_nonempty_stomachs', 'sample_size'] - for field in fields_to_check: - if metrics.get(field) is None: - null_count += 1 - return null_count - - -def print_comparison_table(original_metrics: Dict, improved_metrics: Dict): - """Print a side-by-side comparison of extraction results.""" - - print("\n" + "=" * 80) - print("EXTRACTION COMPARISON: Original vs Improved") - print("=" * 80) - - fields = [ - ('species_name', 'Species Name'), - ('study_location', 'Study Location'), - ('study_date', 'Study Date'), - ('num_empty_stomachs', 'Empty Stomachs'), - ('num_nonempty_stomachs', 'Non-Empty Stomachs'), - ('sample_size', 'Sample Size'), - ('fraction_feeding', 'Fraction Feeding'), - ] - - print(f"\n{'Field':<25} {'Original':<30} {'Improved':<30}") - print("-" * 80) - - for field_key, field_name in fields: - orig_val = original_metrics.get(field_key) - impr_val = improved_metrics.get(field_key) - - orig_str = str(orig_val) if orig_val is not None else "NULL" - impr_str = str(impr_val) if impr_val is not None else "NULL" - - # Highlight improvements - marker = "" - if orig_val is None and impr_val is not None: - marker = " ✓ IMPROVED" - elif orig_val != impr_val and impr_val is not None: - marker = " ✓ CHANGED" - - print(f"{field_name:<25} {orig_str:<30} {impr_str:<30}{marker}") - - print("-" * 80) - - # Summary statistics - orig_null = count_null_fields(original_metrics) - impr_null = count_null_fields(improved_metrics) - - print(f"\nNull Fields: {orig_null}/6 {impr_null}/6") - print(f"Completeness: {((6 - orig_null) / 6) * 100:.1f}% {((6 - impr_null) / 6) * 100:.1f}%") - - if impr_null < orig_null: - print(f"\n✓ IMPROVEMENT: {orig_null - impr_null} fewer null fields!") - elif impr_null > orig_null: - print(f"\n✗ REGRESSION: {impr_null - orig_null} more null fields") - else: - print("\n→ No change in null field count") - - print("=" * 80 + "\n") - - -def main(): - parser = argparse.ArgumentParser(description="Compare original vs improved LLM extraction") - parser.add_argument("text_file", type=str, help="Path to preprocessed text file") - parser.add_argument("pdf_file", type=str, help="Path to original PDF file") - parser.add_argument("--model", type=str, default="llama3.1:8b", help="Model to use for both extractions") - parser.add_argument("--output-dir", type=str, default="data/test_comparison", help="Output directory for comparison results") - - args = parser.parse_args() - - text_path = Path(args.text_file) - pdf_path = Path(args.pdf_file) - - if not text_path.exists(): - print(f"[ERROR] Text file not found: {text_path}") - sys.exit(1) - - if not pdf_path.exists(): - print(f"[ERROR] PDF file not found: {pdf_path}") - sys.exit(1) - - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - print("\n" + "=" * 80) - print("RUNNING EXTRACTION COMPARISON TEST") - print("=" * 80) - print(f"Text file: {text_path.name}") - print(f"PDF file: {pdf_path.name}") - print(f"Model: {args.model}") - print("=" * 80 + "\n") - - # Run original extraction - print("[1/2] Running ORIGINAL extraction...") - print("-" * 80) - - import subprocess - - original_output = output_dir / f"{text_path.stem}_original_results.json" - - try: - result = subprocess.run([sys.executable, "local_llm.py", str(text_path), "--model", args.model, "--output-dir", str(output_dir)], capture_output=True, text=True, timeout=300) - - if result.returncode != 0: - print("[ERROR] Original extraction failed:") - print(result.stderr) - sys.exit(1) - - # Rename output to distinguish it - default_output = output_dir / f"{text_path.stem}_results.json" - if default_output.exists(): - default_output.rename(original_output) - - except subprocess.TimeoutExpired: - print("[ERROR] Original extraction timed out") - sys.exit(1) - except Exception as e: - print(f"[ERROR] Original extraction failed: {e}") - sys.exit(1) - - print("✓ Original extraction complete\n") - - # Run improved extraction - print("[2/2] Running IMPROVED extraction...") - print("-" * 80) - - improved_output = output_dir / f"{text_path.stem}_improved_results.json" - - try: - result = subprocess.run( - [sys.executable, "local_llm_improved.py", str(text_path), "--pdf", str(pdf_path), "--model", args.model, "--output-dir", str(output_dir)], capture_output=True, text=True, timeout=300 - ) - - if result.returncode != 0: - print("[ERROR] Improved extraction failed:") - print(result.stderr) - sys.exit(1) - - # Rename output to distinguish it - default_output = output_dir / f"{text_path.stem}_results.json" - if default_output.exists(): - default_output.rename(improved_output) - - except subprocess.TimeoutExpired: - print("[ERROR] Improved extraction timed out") - sys.exit(1) - except Exception as e: - print(f"[ERROR] Improved extraction failed: {e}") - sys.exit(1) - - print("✓ Improved extraction complete\n") - - # Load and compare results - original_data = load_json_results(str(original_output)) - improved_data = load_json_results(str(improved_output)) - - if not original_data or not improved_data: - print("[ERROR] Could not load results for comparison") - sys.exit(1) - - original_metrics = original_data.get('metrics', {}) - improved_metrics = improved_data.get('metrics', {}) - - # Print comparison - print_comparison_table(original_metrics, improved_metrics) - - # Save comparison report - comparison_report = { - 'text_file': text_path.name, - 'pdf_file': pdf_path.name, - 'model': args.model, - 'original': {'metrics': original_metrics, 'null_count': count_null_fields(original_metrics), 'completeness': f"{((6 - count_null_fields(original_metrics)) / 6) * 100:.1f}%"}, - 'improved': {'metrics': improved_metrics, 'null_count': count_null_fields(improved_metrics), 'completeness': f"{((6 - count_null_fields(improved_metrics)) / 6) * 100:.1f}%"}, - 'improvement': { - 'fields_gained': count_null_fields(original_metrics) - count_null_fields(improved_metrics), - 'is_better': count_null_fields(improved_metrics) < count_null_fields(original_metrics), - }, - } - - report_path = output_dir / f"{text_path.stem}_comparison_report.json" - with open(report_path, 'w', encoding='utf-8') as f: - json.dump(comparison_report, f, indent=2) - - print(f"Detailed comparison report saved to: {report_path}\n") - - -if __name__ == "__main__": - main() From f979970fb7284c1f9fdc347c5dec12c76410d440 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Wed, 8 Apr 2026 18:17:54 -0700 Subject: [PATCH 9/9] fix(test): don't assume file order in test_load_processed_text_reads_files --- tests/test_data_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_data_loader.py b/tests/test_data_loader.py index 89ac320..9ebc47f 100644 --- a/tests/test_data_loader.py +++ b/tests/test_data_loader.py @@ -18,8 +18,8 @@ def test_load_processed_text_reads_files(tmp_path): texts = load_processed_text(str(data_dir)) assert len(texts) == 2 - assert "example 1" in texts[0] - assert "example 2" in texts[1] + assert any("example 1" in t for t in texts) + assert any("example 2" in t for t in texts) def test_load_processed_text_empty_directory(tmp_path):