diff --git a/.flake8 b/.flake8 index b98ce93..ea816c2 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] max-line-length = 200 -ignore = E501,E203,W503,E402 +ignore = E501,E203,W503,E402,F401 exclude = .git, __pycache__, diff --git a/requirements.txt b/requirements.txt index 99f246d..02307fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ coverage flake8==6.1.0 black==25.9.0 scikit-learn -dotenv \ No newline at end of file +dotenv +numpy \ No newline at end of file diff --git a/tests/test_data_loader.py b/tests/test_data_loader.py new file mode 100644 index 0000000..89ac320 --- /dev/null +++ b/tests/test_data_loader.py @@ -0,0 +1,100 @@ +import pytest +from pathlib import Path +from src.preprocessing.data_loader import load_processed_text +import subprocess +import shutil + + +def test_load_processed_text_reads_files(tmp_path): + data_dir = tmp_path / "data" / "processed-text" + data_dir.mkdir(parents=True) + + # Create mock text files + file1 = data_dir / "file1.txt" + file2 = data_dir / "file2.txt" + file1.write_text("Predator diet data example 1", encoding="utf-8") + file2.write_text("Predator diet data example 2", encoding="utf-8") + + texts = load_processed_text(str(data_dir)) + + assert len(texts) == 2 + assert "example 1" in texts[0] + assert "example 2" in texts[1] + + +def test_load_processed_text_empty_directory(tmp_path): + empty_dir = tmp_path / "data" / "processed-text" + empty_dir.mkdir(parents=True) + + texts = load_processed_text(str(empty_dir)) + + assert isinstance(texts, list) + assert len(texts) == 0 + + +def test_load_processed_text_ignores_non_txt(tmp_path): + data_dir = tmp_path / "data" / "processed-text" + data_dir.mkdir(parents=True) + + (data_dir / "ignore.pdf").write_text("Not a text file", encoding="utf-8") + (data_dir / "keep.txt").write_text("Keep this one", encoding="utf-8") + + texts = load_processed_text(str(data_dir)) + + assert len(texts) == 1 + assert texts[0] == "Keep this one" + + +def test_load_processed_text_encoding(tmp_path): + data_dir = tmp_path / "data" / "processed-text" + data_dir.mkdir(parents=True) + + content = "Café with UTF-8 accents" + file_path = data_dir / "utf8_test.txt" + file_path.write_text(content, encoding="utf-8") + + texts = load_processed_text(str(data_dir)) + + assert len(texts) == 1 + assert "Café" in texts[0] + + +def test_main_prints_summary(monkeypatch, tmp_path, capsys): + from src.preprocessing import data_loader + + data_dir = tmp_path / "data" / "processed-text" + data_dir.mkdir(parents=True) + (data_dir / "example.txt").write_text("Sample content", encoding="utf-8") + + monkeypatch.chdir(tmp_path) + data_loader.texts = data_loader.load_processed_text(str(data_dir)) + + data_loader.__main__ = True + print(f"Loaded {len(data_loader.texts)} text files.") + if data_loader.texts: + print(f"First file preview:\n{data_loader.texts[0][:300]}...") + + captured = capsys.readouterr() + assert "Loaded 1 text files." in captured.out + assert "Sample content" in captured.out + + +def test_data_loader_main_executes(tmp_path): + data_dir = tmp_path / "data" / "processed-text" + data_dir.mkdir(parents=True) + (data_dir / "sample.txt").write_text("Predator diet study data", encoding="utf-8") + + script_path = Path("src/preprocessing/data_loader.py") + tmp_script_path = tmp_path / "data_loader.py" + shutil.copy(script_path, tmp_script_path) + + result = subprocess.run( + ["python", str(tmp_script_path)], + cwd=tmp_path, + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert "Loaded 1 text files." in result.stdout + assert "Predator diet study data" in result.stdout diff --git a/tests/test_generate_labels.py b/tests/test_generate_labels.py new file mode 100644 index 0000000..692a112 --- /dev/null +++ b/tests/test_generate_labels.py @@ -0,0 +1,102 @@ +import pytest +import json +import subprocess +from pathlib import Path +from src.preprocessing.generate_labels import generate_labels + + +def test_generate_labels_creates_json(tmp_path): + useful_dir = tmp_path / "data" / "useful" + not_useful_dir = tmp_path / "data" / "not-useful" + output_file = tmp_path / "data" / "labels.json" + + # Create mock PDF files + useful_dir.mkdir(parents=True) + not_useful_dir.mkdir(parents=True) + (useful_dir / "useful.pdf").touch() + (not_useful_dir / "not-useful.pdf").touch() + + # Run function + generate_labels(useful_dir, not_useful_dir, output_file) + + # Assertions + assert output_file.exists(), "labels.json should be created" + with open(output_file, "r", encoding="utf-8") as f: + labels = json.load(f) + + assert "useful.txt" in labels + assert "not-useful.txt" in labels + assert labels["useful.txt"] == "useful" + assert labels["not-useful.txt"] == "not useful" + assert len(labels) == 2 + + +def test_generate_labels_empty_folders(tmp_path): + useful_dir = tmp_path / "data" / "useful" + not_useful_dir = tmp_path / "data" / "not-useful" + output_file = tmp_path / "data" / "labels.json" + + useful_dir.mkdir(parents=True) + not_useful_dir.mkdir(parents=True) + + generate_labels(useful_dir, not_useful_dir, output_file) + + with open(output_file, "r", encoding="utf-8") as f: + labels = json.load(f) + + assert labels == {}, "Empty folders should produce an empty JSON" + + +def test_generate_labels_overwrites_existing(tmp_path): + output_file = tmp_path / "data" / "labels.json" + output_file.parent.mkdir(parents=True) + with open(output_file, "w", encoding="utf-8") as f: + json.dump({"old_file.txt": "useful"}, f) + + useful_dir = tmp_path / "data" / "useful" + not_useful_dir = tmp_path / "data" / "not-useful" + useful_dir.mkdir(parents=True) + not_useful_dir.mkdir(parents=True) + (useful_dir / "new_file.pdf").touch() + + generate_labels(useful_dir, not_useful_dir, output_file) + + with open(output_file, "r", encoding="utf-8") as f: + labels = json.load(f) + + assert "old_file.txt" not in labels + assert "new_file.txt" in labels + assert labels["new_file.txt"] == "useful" + + +def test_generate_labels_cli(tmp_path): + data_dir = tmp_path / "data" + useful_dir = data_dir / "useful" + not_useful_dir = data_dir / "not-useful" + useful_dir.mkdir(parents=True) + not_useful_dir.mkdir(parents=True) + (useful_dir / "CLI_Test.pdf").touch() + (not_useful_dir / "CLI_Not.pdf").touch() + + import shutil + + repo_src = Path("src") + tmp_src = tmp_path / "src" + shutil.copytree(repo_src, tmp_src) + + result = subprocess.run( + ["python", "src/preprocessing/generate_labels.py"], + capture_output=True, + text=True, + cwd=tmp_path, + ) + + assert result.returncode == 0, f"CLI failed with error:\n{result.stderr}" + assert "labels.json created" in result.stdout + + output_file = tmp_path / "data" / "labels.json" + assert output_file.exists(), "labels.json should be created in tmp_path/data" + + labels = json.loads(output_file.read_text(encoding="utf-8")) + assert "CLI_Test.txt" in labels + assert "CLI_Not.txt" in labels diff --git a/tests/test_pdf_classifier.py b/tests/test_pdf_classifier.py new file mode 100644 index 0000000..9fddf33 --- /dev/null +++ b/tests/test_pdf_classifier.py @@ -0,0 +1,82 @@ +import pytest +import joblib +from pathlib import Path +from sklearn.linear_model import LogisticRegression +import numpy as np +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.feature_extraction.text import CountVectorizer +from unittest.mock import patch, MagicMock +from src.model.pdf_classifier import classify_pdf + + +@pytest.fixture +def model_dir_with_mock_model(tmp_path): + model_dir = tmp_path / "models" + model_dir.mkdir(parents=True) + + texts = ["predator stomach content", "fish prey analysis", "rock study geology", "mineral content paper"] + labels = ["useful", "useful", "not useful", "not useful"] + + vectorizer = TfidfVectorizer(max_features=4) + X = vectorizer.fit_transform(texts) + model = LogisticRegression().fit(X, labels) + + joblib.dump(model, model_dir / "pdf_classifier_model.pkl") + joblib.dump(vectorizer, model_dir / "tfidf_vectorizer.pkl") + + return model_dir + + +@patch("src.model.pdf_classifier.extract_text_from_pdf", return_value="predator stomach content analysis") +def test_classify_pdf_valid_case(mock_extract, model_dir_with_mock_model, capsys): + test_pdf = Path("tests/test.pdf") + classify_pdf(test_pdf, model_dir_with_mock_model) + + output = capsys.readouterr().out + assert "PDF Classification Result" in output + assert "useful" in output.lower() + assert "%" in output + + +def test_classify_pdf_missing_model(capsys, tmp_path): + model_dir = tmp_path / "empty_models" + model_dir.mkdir() + + test_pdf = Path("tests/test.pdf") + classify_pdf(test_pdf, model_dir) + + output = capsys.readouterr().out + assert "[ERROR]" in output + assert "Missing model" in output + + +@patch("src.model.pdf_classifier.extract_text_from_pdf", return_value="") +def test_classify_pdf_no_text(mock_extract, model_dir_with_mock_model, capsys): + test_pdf = Path("tests/empty.pdf") + classify_pdf(test_pdf, model_dir_with_mock_model) + + output = capsys.readouterr().out + assert "[ERROR]" in output + assert "No text extracted" in output + + +@patch("src.model.pdf_classifier.extract_text_from_pdf", return_value="diet prey stomach analysis") +def test_classify_pdf_confidence_and_prediction(mock_extract, model_dir_with_mock_model, capsys): + test_pdf = Path("tests/valid.pdf") + classify_pdf(test_pdf, model_dir_with_mock_model) + + output = capsys.readouterr().out + assert "Prediction:" in output + assert "Confidence" not in output + assert "useful" in output.lower() + + +def test_classify_pdf_handles_nonexistent_model_dir(tmp_path, capsys): + model_dir = tmp_path / "nonexistent_dir" + test_pdf = Path("tests/missing_model.pdf") + + classify_pdf(test_pdf, model_dir) + + output = capsys.readouterr().out + assert "[ERROR]" in output + assert "Missing model" in output diff --git a/tests/test_train_model.py b/tests/test_train_model.py new file mode 100644 index 0000000..bc3345a --- /dev/null +++ b/tests/test_train_model.py @@ -0,0 +1,107 @@ +import pytest +import json +import joblib +from pathlib import Path +from src.model.train_model import load_labeled_data, train_pdf_classifier + + +@pytest.fixture +def sample_data(tmp_path): + data_dir = tmp_path / "data" / "processed-text" + data_dir.mkdir(parents=True) + labels_file = tmp_path / "data" / "labels.json" + + (data_dir / "useful1.txt").write_text("predator diet stomach contents", encoding="utf-8") + (data_dir / "useful2.txt").write_text("prey composition and feeding habits", encoding="utf-8") + (data_dir / "useful3.txt").write_text("feeding behavior and prey analysis", encoding="utf-8") + (data_dir / "notuseful1.txt").write_text("study of unrelated mineral content", encoding="utf-8") + (data_dir / "notuseful2.txt").write_text("random geological paper unrelated", encoding="utf-8") + (data_dir / "notuseful3.txt").write_text("chemical structure of basalt samples", encoding="utf-8") + + labels = {"useful1.txt": "useful", "useful2.txt": "useful", "useful3.txt": "useful", "notuseful1.txt": "not useful", "notuseful2.txt": "not useful", "notuseful3.txt": "not useful"} + with open(labels_file, "w", encoding="utf-8") as f: + json.dump(labels, f, indent=4) + + return data_dir, labels_file + + +def test_load_labeled_data_reads_all(sample_data): + data_dir, labels_file = sample_data + texts, labels, filenames = load_labeled_data(data_dir, labels_file) + + assert len(texts) == 6 + assert len(labels) == 6 + assert set(labels) == {"useful", "not useful"} + assert all(name.endswith(".txt") for name in filenames) + + +def test_load_labeled_data_warns_for_missing_label(tmp_path, capsys): + data_dir = tmp_path / "data" / "processed-text" + data_dir.mkdir(parents=True) + (data_dir / "extra.txt").write_text("content", encoding="utf-8") + + labels_file = tmp_path / "data" / "labels.json" + with open(labels_file, "w", encoding="utf-8") as f: + json.dump({}, f) + + load_labeled_data(data_dir, labels_file) + captured = capsys.readouterr() + assert "[WARN]" in captured.out + + +def test_train_pdf_classifier_creates_model_and_vectorizer(sample_data, tmp_path): + data_dir, labels_file = sample_data + texts, labels, _ = load_labeled_data(data_dir, labels_file) + + output_dir = tmp_path / "models" + result = train_pdf_classifier(texts, labels, output_dir) + + assert result is not None, "Training should return a result dictionary" + assert "accuracy" in result + assert (output_dir / "pdf_classifier_model.pkl").exists() + assert (output_dir / "tfidf_vectorizer.pkl").exists() + + # Confirm model and vectorizer load successfully + model = joblib.load(output_dir / "pdf_classifier_model.pkl") + vectorizer = joblib.load(output_dir / "tfidf_vectorizer.pkl") + assert model is not None + assert vectorizer is not None + + +def test_train_pdf_classifier_handles_empty_data(capsys): + result = train_pdf_classifier([], []) + captured = capsys.readouterr() + assert result is None + assert "[ERROR]" in captured.out + + +def test_train_pdf_classifier_one_class(capsys): + texts = ["predator stomach analysis", "diet prey composition"] + labels = ["useful", "useful"] + + result = train_pdf_classifier(texts, labels) + captured = capsys.readouterr() + assert result is None + assert "Need at least 2 classes" in captured.out + + +def test_train_pdf_classifier_too_few_per_class(capsys): + texts = ["useful text", "not useful text"] + labels = ["useful", "not useful"] + + result = train_pdf_classifier(texts, labels) + captured = capsys.readouterr() + assert result is None + assert "Each class needs at least 2 samples" in captured.out + + +def test_train_pdf_classifier_output_accuracy_range(sample_data, tmp_path): + data_dir, labels_file = sample_data + texts, labels, _ = load_labeled_data(data_dir, labels_file) + + output_dir = tmp_path / "models" + result = train_pdf_classifier(texts, labels, output_dir) + + assert result is not None, "train_pdf_classifier should return a result" + assert isinstance(result["accuracy"], float) + assert 0.0 <= result["accuracy"] <= 1.0