diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..0bd1b1f --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +max-line-length = 200 +ignore = E501,E203,W503 +exclude = + .git, + __pycache__, + build, + dist, + venv, + .venv, + .mypy_cache, + htmlcov diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 8818599..bb2c90d 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -1,10 +1,6 @@ name: PDF Extraction CI -on: - push: - branches: [ working-software ] - pull_request: - branches: [ working-software ] +on: [push, pull_request] jobs: run: @@ -24,15 +20,27 @@ jobs: sudo apt-get update sudo apt-get install -y tesseract-ocr python -m pip install --upgrade pip - pip install pymupdf pytesseract pillow pytest coverage + pip install -r requirements.txt + # Code quality checks + - name: Check code formatting (Black) + run: | + echo "Checking code format with Black..." + black --check src tests + + - name: Run linter (Flake8) + run: | + echo "Running flake8 linter..." + flake8 src/ tests/ + + # Functional validation - name: Run PDF extraction script run: | mkdir -p data/processed-text INPUT_PDF="tests/test.pdf" python src/preprocessing/pdf_text_extraction.py "$INPUT_PDF" - - name: Run generic output tests + - name: Validate extraction output run: | INPUT_PDF="tests/test.pdf" OUTPUT_FILE="data/processed-text/$(basename "$INPUT_PDF" .pdf).txt" @@ -40,60 +48,35 @@ jobs: failures=0 # Test 1: Output file exists - if [ -f "$OUTPUT_FILE" ]; then - echo "Test 1 passed: $OUTPUT_FILE exists" - else - echo "Test 1 failed: $OUTPUT_FILE missing" - failures=$((failures+1)) - fi + [ -f "$OUTPUT_FILE" ] && echo "Test 1 passed: file exists" || { echo "Test 1 failed: missing"; failures=$((failures+1)); } # Test 2: File is not empty - if [ -s "$OUTPUT_FILE" ]; then - echo "Test 2 passed: file is not empty" - else - echo "Test 2 failed: file is empty" - failures=$((failures+1)) - fi - - # Test 3: At least 1 line + [ -s "$OUTPUT_FILE" ] && echo "Test 2 passed: not empty" || { echo "Test 2 failed: empty"; failures=$((failures+1)); } + + # Test 3: At least one line line_count=$(wc -l < "$OUTPUT_FILE") - if [ "$line_count" -ge 1 ]; then - echo "Test 3 passed: file has $line_count lines" - else - echo "Test 3 failed: file has no lines" - failures=$((failures+1)) - fi - - # Test 4: At least 1 word + [ "$line_count" -ge 1 ] && echo "Test 3 passed: $line_count lines" || { echo "Test 3 failed: no lines"; failures=$((failures+1)); } + + # Test 4: At least one word word_count=$(wc -w < "$OUTPUT_FILE") - if [ "$word_count" -ge 1 ]; then - echo "Test 4 passed: at least 1 word extracted" - else - echo "Test 4 failed: no words extracted" - failures=$((failures+1)) - fi - - # Test 5: File size reasonable (>=10 bytes) + [ "$word_count" -ge 1 ] && echo "Test 4 passed: $word_count words" || { echo "Test 4 failed: no words"; failures=$((failures+1)); } + + # Test 5: Reasonable file size file_size=$(stat -c%s "$OUTPUT_FILE") - if [ "$file_size" -ge 10 ]; then - echo "Test 5 passed: file size is $file_size bytes" - else - echo "Test 5 failed: file too small ($file_size bytes)" - failures=$((failures+1)) - fi - - echo "Generic tests summary: $total_tests total, $failures failures" - if [ $failures -gt 0 ]; then - exit 1 - fi - - - name: Run Python unit tests with coverage + [ "$file_size" -ge 10 ] && echo "Test 5 passed: size ${file_size} bytes" || { echo "Test 5 failed: too small (${file_size})"; failures=$((failures+1)); } + + echo "Summary: $total_tests tests, $failures failed" + [ $failures -eq 0 ] + + # Unit testing and coverage + - name: Run unit tests with coverage run: | + echo "Running pytest with coverage..." coverage run -m pytest tests/ coverage report -m coverage html - - name: Upload coverage report artifact + - name: Upload coverage report uses: actions/upload-artifact@v4 with: name: coverage-report @@ -103,5 +86,5 @@ jobs: run: | INPUT_PDF="tests/test.pdf" OUTPUT_FILE="data/processed-text/$(basename "$INPUT_PDF" .pdf).txt" - echo "Extracted text from $OUTPUT_FILE:" - cat "$OUTPUT_FILE" || echo "No output file found" + echo " Extracted text from $OUTPUT_FILE:" + cat "$OUTPUT_FILE" || echo " No output file found" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..42da6f0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 200 +target-version = ["py311"] +skip-string-normalization = true +exclude = ''' +/( + \.git + | __pycache__ + | build + | dist + | venv + | \.venv + | \.mypy_cache + | htmlcov +)/ +''' + +[tool.pytest.ini_options] +addopts = "-ra --disable-warnings" +testpaths = [ + "tests" +] +filterwarnings = [ + "ignore::DeprecationWarning" +] + +[tool.coverage.run] +branch = true +source = ["src"] +omit = ["tests/*"] + +[tool.coverage.report] +show_missing = true +skip_covered = true +precision = 1 diff --git a/requirements.txt b/requirements.txt index a74ac47..b8e9e82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ pymupdf pillow -pytesseract \ No newline at end of file +pytesseract +pytest +coverage +flake8==6.1.0 +black==25.9.0 \ No newline at end of file diff --git a/src/preprocessing/pdf_text_extraction.py b/src/preprocessing/pdf_text_extraction.py index c2d2ff0..cb0295f 100644 --- a/src/preprocessing/pdf_text_extraction.py +++ b/src/preprocessing/pdf_text_extraction.py @@ -1,4 +1,4 @@ -""" PDF Text Extraction Script +"""PDF Text Extraction Script Usage: python pdf_to_text.py path/to/input.pdf @@ -17,6 +17,7 @@ from pathlib import Path import sys + def extract_text_from_pdf(pdf_path: str) -> str: text = [] try: @@ -28,13 +29,13 @@ def extract_text_from_pdf(pdf_path: str) -> str: # Clean out null bytes or UTF-16 artifacts if "\x00" in page_text: page_text = page_text.replace("\x00", "") - + # If the page is mostly empty, treat as image and use OCR if not page_text.strip(): pix = page.get_pixmap(dpi=300) img = Image.open(io.BytesIO(pix.tobytes("png"))) page_text = pytesseract.image_to_string(img) - + text.append(page_text) except Exception as e: print(f"[ERROR] Failed to extract text from {pdf_path}: {e}", file=sys.stderr) diff --git a/tests/test_pdf_extraction.py b/tests/test_pdf_extraction.py index 2c12727..d59e1e7 100644 --- a/tests/test_pdf_extraction.py +++ b/tests/test_pdf_extraction.py @@ -1,8 +1,91 @@ -from src.preprocessing.pdf_text_extraction import extract_text_from_pdf +import pytest +import subprocess +import fitz from pathlib import Path +import sys +from src.preprocessing.pdf_text_extraction import extract_text_from_pdf, save_to_file, main + def test_extract_text_exists(): pdf_path = Path("tests/test.pdf") text = extract_text_from_pdf(str(pdf_path)) assert text is not None - assert len(text) > 0 \ No newline at end of file + assert len(text) > 0 + + +def test_save_to_file_creates_output(tmp_path): + output_path = tmp_path / "output.txt" + text = "Sample text" + save_to_file(text, output_path) + assert output_path.exists() + assert output_path.read_text(encoding="utf-8") == text + + +def test_main_handles_missing_pdf(monkeypatch, tmp_path): + fake_pdf = tmp_path / "nonexistent.pdf" + test_args = ["pdf_to_text.py", str(fake_pdf)] + monkeypatch.setattr(sys, "argv", test_args) + with pytest.raises(SystemExit): + main() + + +def test_extract_text_invalid_path(tmp_path): + invalid_pdf = tmp_path / "does_not_exist.pdf" + result = extract_text_from_pdf(str(invalid_pdf)) + assert result == "" + + +def test_extract_text_with_ocr(tmp_path): + pdf_path = tmp_path / "image_only.pdf" + + # Create image-only PDF + doc = fitz.open() + page = doc.new_page() + # Draw white rectangle (no text layer) + rect = fitz.Rect(0, 0, 100, 100) + page.insert_textbox(rect, "") # empty text + pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 100, 100)) + page.insert_image(rect, pixmap=pix) + doc.save(pdf_path) + doc.close() + + text = extract_text_from_pdf(str(pdf_path)) + assert isinstance(text, str) + # Empty string is acceptable if OCR yields nothing, but must not crash + assert text is not None + + +def test_save_to_file(tmp_path): + output_file = tmp_path / "output.txt" + save_to_file("hello world", output_file) + assert output_file.exists() + contents = output_file.read_text() + assert "hello" in contents + + +def test_save_to_file_permission_error(monkeypatch, tmp_path): + output_file = tmp_path / "readonly.txt" + output_file.touch() + output_file.chmod(0o400) + + # Monkeypatch print to avoid cluttering stderr + monkeypatch.setattr("sys.stderr", open("/dev/null", "w")) + + # Should not raise exception even if write fails + save_to_file("data", output_file) + + +def test_main_cli(tmp_path): + input_pdf = "tests/test.pdf" + output_dir = Path("data/processed-text") + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "test.txt" + + result = subprocess.run( + ["python", "src/preprocessing/pdf_text_extraction.py", input_pdf], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert output_file.exists() + assert output_file.stat().st_size > 0