Linter and Code Formatter Configuration

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+max-line-length = 200
+ignore = E501,E203,W503
+exclude =
+    .git,
+    __pycache__,
+    build,
+    dist,
+    venv,
+    .venv,
+    .mypy_cache,
+    htmlcov

.github/workflows/working_sw.yml

@@ -1,10 +1,6 @@
 name: PDF Extraction CI
 
-on:
-  push:
-    branches: [ working-software ]
-  pull_request:
-    branches: [ working-software ]
+on: [push, pull_request]
 
 jobs:
   run:
@@ -24,76 +20,63 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y tesseract-ocr
           python -m pip install --upgrade pip
-          pip install pymupdf pytesseract pillow pytest coverage
+          pip install -r requirements.txt
 
+      # Code quality checks
+      - name: Check code formatting (Black)
+        run: |
+          echo "Checking code format with Black..."
+          black --check src tests
+
+      - name: Run linter (Flake8)
+        run: |
+          echo "Running flake8 linter..."
+          flake8 src/ tests/
+
+      # Functional validation
       - name: Run PDF extraction script
         run: |
           mkdir -p data/processed-text
           INPUT_PDF="tests/test.pdf"
           python src/preprocessing/pdf_text_extraction.py "$INPUT_PDF"
 
-      - name: Run generic output tests
+      - name: Validate extraction output
         run: |
           INPUT_PDF="tests/test.pdf"
           OUTPUT_FILE="data/processed-text/$(basename "$INPUT_PDF" .pdf).txt"
           total_tests=5
           failures=0
 
           # Test 1: Output file exists
-          if [ -f "$OUTPUT_FILE" ]; then
-            echo "Test 1 passed: $OUTPUT_FILE exists"
-          else
-            echo "Test 1 failed: $OUTPUT_FILE missing"
-            failures=$((failures+1))
-          fi
+          [ -f "$OUTPUT_FILE" ] && echo "Test 1 passed: file exists" || { echo "Test 1 failed: missing"; failures=$((failures+1)); }
 
           # Test 2: File is not empty
-          if [ -s "$OUTPUT_FILE" ]; then
-            echo "Test 2 passed: file is not empty"
-          else
-            echo "Test 2 failed: file is empty"
-            failures=$((failures+1))
-          fi
-
-          # Test 3: At least 1 line
+          [ -s "$OUTPUT_FILE" ] && echo "Test 2 passed: not empty" || { echo "Test 2 failed: empty"; failures=$((failures+1)); }
+
+          # Test 3: At least one line
           line_count=$(wc -l < "$OUTPUT_FILE")
-          if [ "$line_count" -ge 1 ]; then
-            echo "Test 3 passed: file has $line_count lines"
-          else
-            echo "Test 3 failed: file has no lines"
-            failures=$((failures+1))
-          fi
-
-          # Test 4: At least 1 word
+          [ "$line_count" -ge 1 ] && echo "Test 3 passed: $line_count lines" || { echo "Test 3 failed: no lines"; failures=$((failures+1)); }
+
+          # Test 4: At least one word
           word_count=$(wc -w < "$OUTPUT_FILE")
-          if [ "$word_count" -ge 1 ]; then
-            echo "Test 4 passed: at least 1 word extracted"
-          else
-            echo "Test 4 failed: no words extracted"
-            failures=$((failures+1))
-          fi
-
-          # Test 5: File size reasonable (>=10 bytes)
+          [ "$word_count" -ge 1 ] && echo "Test 4 passed: $word_count words" || { echo "Test 4 failed: no words"; failures=$((failures+1)); }
+
+          # Test 5: Reasonable file size
           file_size=$(stat -c%s "$OUTPUT_FILE")
-          if [ "$file_size" -ge 10 ]; then
-            echo "Test 5 passed: file size is $file_size bytes"
-          else
-            echo "Test 5 failed: file too small ($file_size bytes)"
-            failures=$((failures+1))
-          fi
-
-          echo "Generic tests summary: $total_tests total, $failures failures"
-          if [ $failures -gt 0 ]; then
-            exit 1
-          fi
-
-      - name: Run Python unit tests with coverage
+          [ "$file_size" -ge 10 ] && echo "Test 5 passed: size ${file_size} bytes" || { echo "Test 5 failed: too small (${file_size})"; failures=$((failures+1)); }
+
+          echo "Summary: $total_tests tests, $failures failed"
+          [ $failures -eq 0 ]
+
+      # Unit testing and coverage
+      - name: Run unit tests with coverage
         run: |
+          echo "Running pytest with coverage..."
           coverage run -m pytest tests/
           coverage report -m
           coverage html
 
-      - name: Upload coverage report artifact
+      - name: Upload coverage report
         uses: actions/upload-artifact@v4
         with:
           name: coverage-report
@@ -103,5 +86,5 @@ jobs:
         run: |
           INPUT_PDF="tests/test.pdf"
           OUTPUT_FILE="data/processed-text/$(basename "$INPUT_PDF" .pdf).txt"
-          echo "Extracted text from $OUTPUT_FILE:"
-          cat "$OUTPUT_FILE" || echo "No output file found"
+          echo " Extracted text from $OUTPUT_FILE:"
+          cat "$OUTPUT_FILE" || echo " No output file found"

pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 200
+target-version = ["py311"]
+skip-string-normalization = true
+exclude = '''
+/(
+    \.git
+  | __pycache__
+  | build
+  | dist
+  | venv
+  | \.venv
+  | \.mypy_cache
+  | htmlcov
+)/
+'''
+
+[tool.pytest.ini_options]
+addopts = "-ra --disable-warnings"
+testpaths = [
+    "tests"
+]
+filterwarnings = [
+    "ignore::DeprecationWarning"
+]
+
+[tool.coverage.run]
+branch = true
+source = ["src"]
+omit = ["tests/*"]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+precision = 1

requirements.txt

@@ -1,3 +1,7 @@
 pymupdf
 pillow
-pytesseract
+pytesseract
+pytest
+coverage
+flake8==6.1.0
+black==25.9.0

src/preprocessing/pdf_text_extraction.py

@@ -1,4 +1,4 @@
-""" PDF Text Extraction Script
+"""PDF Text Extraction Script
 
 Usage:
     python pdf_to_text.py path/to/input.pdf
@@ -17,6 +17,7 @@
 from pathlib import Path
 import sys
 
+
 def extract_text_from_pdf(pdf_path: str) -> str:
     text = []
     try:
@@ -28,13 +29,13 @@ def extract_text_from_pdf(pdf_path: str) -> str:
                 # Clean out null bytes or UTF-16 artifacts
                 if "\x00" in page_text:
                     page_text = page_text.replace("\x00", "")
-                
+
                 # If the page is mostly empty, treat as image and use OCR
                 if not page_text.strip():
                     pix = page.get_pixmap(dpi=300)
                     img = Image.open(io.BytesIO(pix.tobytes("png")))
                     page_text = pytesseract.image_to_string(img)
-                
+
                 text.append(page_text)
     except Exception as e:
         print(f"[ERROR] Failed to extract text from {pdf_path}: {e}", file=sys.stderr)

tests/test_pdf_extraction.py

@@ -1,8 +1,91 @@
-from src.preprocessing.pdf_text_extraction import extract_text_from_pdf
+import pytest
+import subprocess
+import fitz
 from pathlib import Path
+import sys
+from src.preprocessing.pdf_text_extraction import extract_text_from_pdf, save_to_file, main
+
 
 def test_extract_text_exists():
     pdf_path = Path("tests/test.pdf")
     text = extract_text_from_pdf(str(pdf_path))
     assert text is not None
-    assert len(text) > 0
+    assert len(text) > 0
+
+
+def test_save_to_file_creates_output(tmp_path):
+    output_path = tmp_path / "output.txt"
+    text = "Sample text"
+    save_to_file(text, output_path)
+    assert output_path.exists()
+    assert output_path.read_text(encoding="utf-8") == text
+
+
+def test_main_handles_missing_pdf(monkeypatch, tmp_path):
+    fake_pdf = tmp_path / "nonexistent.pdf"
+    test_args = ["pdf_to_text.py", str(fake_pdf)]
+    monkeypatch.setattr(sys, "argv", test_args)
+    with pytest.raises(SystemExit):
+        main()
+
+
+def test_extract_text_invalid_path(tmp_path):
+    invalid_pdf = tmp_path / "does_not_exist.pdf"
+    result = extract_text_from_pdf(str(invalid_pdf))
+    assert result == ""
+
+
+def test_extract_text_with_ocr(tmp_path):
+    pdf_path = tmp_path / "image_only.pdf"
+
+    # Create image-only PDF
+    doc = fitz.open()
+    page = doc.new_page()
+    # Draw white rectangle (no text layer)
+    rect = fitz.Rect(0, 0, 100, 100)
+    page.insert_textbox(rect, "")  # empty text
+    pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 100, 100))
+    page.insert_image(rect, pixmap=pix)
+    doc.save(pdf_path)
+    doc.close()
+
+    text = extract_text_from_pdf(str(pdf_path))
+    assert isinstance(text, str)
+    # Empty string is acceptable if OCR yields nothing, but must not crash
+    assert text is not None
+
+
+def test_save_to_file(tmp_path):
+    output_file = tmp_path / "output.txt"
+    save_to_file("hello world", output_file)
+    assert output_file.exists()
+    contents = output_file.read_text()
+    assert "hello" in contents
+
+
+def test_save_to_file_permission_error(monkeypatch, tmp_path):
+    output_file = tmp_path / "readonly.txt"
+    output_file.touch()
+    output_file.chmod(0o400)
+
+    # Monkeypatch print to avoid cluttering stderr
+    monkeypatch.setattr("sys.stderr", open("/dev/null", "w"))
+
+    # Should not raise exception even if write fails
+    save_to_file("data", output_file)
+
+
+def test_main_cli(tmp_path):
+    input_pdf = "tests/test.pdf"
+    output_dir = Path("data/processed-text")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    output_file = output_dir / "test.txt"
+
+    result = subprocess.run(
+        ["python", "src/preprocessing/pdf_text_extraction.py", input_pdf],
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0
+    assert output_file.exists()
+    assert output_file.stat().st_size > 0