From 72c4d6b8de1ead56b33414b0f9457648fb188bc7 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 19:50:25 -0800 Subject: [PATCH 1/8] Implement linter (flake8) and code formatter (black) --- .github/workflows/working_sw.yml | 90 ++++++++++-------------- pyproject.toml | 44 ++++++++++++ requirements.txt | 6 +- src/preprocessing/pdf_text_extraction.py | 5 +- 4 files changed, 89 insertions(+), 56 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 8818599..15a77f1 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -1,10 +1,6 @@ name: PDF Extraction CI -on: - push: - branches: [ working-software ] - pull_request: - branches: [ working-software ] +on: [push, pull_request] jobs: run: @@ -24,15 +20,28 @@ jobs: sudo apt-get update sudo apt-get install -y tesseract-ocr python -m pip install --upgrade pip - pip install pymupdf pytesseract pillow pytest coverage + pip install -r requirements.txt + pip install black flake8 coverage pytest + # Code quality checks + - name: Run linter (Flake8) + run: | + echo "Running flake8 linter..." + flake8 src/ tests/ + + - name: Check code formatting (Black) + run: | + echo "Checking code format with Black..." + black --check src tests + + # Functional validation - name: Run PDF extraction script run: | mkdir -p data/processed-text INPUT_PDF="tests/test.pdf" python src/preprocessing/pdf_text_extraction.py "$INPUT_PDF" - - name: Run generic output tests + - name: Validate extraction output run: | INPUT_PDF="tests/test.pdf" OUTPUT_FILE="data/processed-text/$(basename "$INPUT_PDF" .pdf).txt" @@ -40,60 +49,35 @@ jobs: failures=0 # Test 1: Output file exists - if [ -f "$OUTPUT_FILE" ]; then - echo "Test 1 passed: $OUTPUT_FILE exists" - else - echo "Test 1 failed: $OUTPUT_FILE missing" - failures=$((failures+1)) - fi + [ -f "$OUTPUT_FILE" ] && echo "Test 1 passed: file exists" || { echo "Test 1 failed: missing"; failures=$((failures+1)); } # Test 2: File is not empty - if [ -s "$OUTPUT_FILE" ]; then - echo "Test 2 passed: file is not empty" - else - echo "Test 2 failed: file is empty" - failures=$((failures+1)) - fi - - # Test 3: At least 1 line + [ -s "$OUTPUT_FILE" ] && echo "Test 2 passed: not empty" || { echo "Test 2 failed: empty"; failures=$((failures+1)); } + + # Test 3: At least one line line_count=$(wc -l < "$OUTPUT_FILE") - if [ "$line_count" -ge 1 ]; then - echo "Test 3 passed: file has $line_count lines" - else - echo "Test 3 failed: file has no lines" - failures=$((failures+1)) - fi - - # Test 4: At least 1 word + [ "$line_count" -ge 1 ] && echo "Test 3 passed: $line_count lines" || { echo "Test 3 failed: no lines"; failures=$((failures+1)); } + + # Test 4: At least one word word_count=$(wc -w < "$OUTPUT_FILE") - if [ "$word_count" -ge 1 ]; then - echo "Test 4 passed: at least 1 word extracted" - else - echo "Test 4 failed: no words extracted" - failures=$((failures+1)) - fi - - # Test 5: File size reasonable (>=10 bytes) + [ "$word_count" -ge 1 ] && echo "Test 4 passed: $word_count words" || { echo "Test 4 failed: no words"; failures=$((failures+1)); } + + # Test 5: Reasonable file size file_size=$(stat -c%s "$OUTPUT_FILE") - if [ "$file_size" -ge 10 ]; then - echo "Test 5 passed: file size is $file_size bytes" - else - echo "Test 5 failed: file too small ($file_size bytes)" - failures=$((failures+1)) - fi - - echo "Generic tests summary: $total_tests total, $failures failures" - if [ $failures -gt 0 ]; then - exit 1 - fi - - - name: Run Python unit tests with coverage + [ "$file_size" -ge 10 ] && echo "Test 5 passed: size ${file_size} bytes" || { echo "Test 5 failed: too small (${file_size})"; failures=$((failures+1)); } + + echo "Summary: $total_tests tests, $failures failed" + [ $failures -eq 0 ] + + # Unit testing and coverage + - name: Run unit tests with coverage run: | + echo "Running pytest with coverage..." coverage run -m pytest tests/ coverage report -m coverage html - - name: Upload coverage report artifact + - name: Upload coverage report uses: actions/upload-artifact@v4 with: name: coverage-report @@ -103,5 +87,5 @@ jobs: run: | INPUT_PDF="tests/test.pdf" OUTPUT_FILE="data/processed-text/$(basename "$INPUT_PDF" .pdf).txt" - echo "Extracted text from $OUTPUT_FILE:" - cat "$OUTPUT_FILE" || echo "No output file found" + echo " Extracted text from $OUTPUT_FILE:" + cat "$OUTPUT_FILE" || echo " No output file found" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d89789b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 100 +target-version = ["py311"] +skip-string-normalization = true + +[tool.flake8] +max-line-length = 100 +ignore = [ + "E203", # whitespace before ':' + "W503", # line break before binary operator +] +exclude = [ + ".git", + "__pycache__", + "build", + "dist", + "venv", + ".venv", + ".mypy_cache", + "htmlcov", +] + +[tool.pytest.ini_options] +addopts = "-ra --disable-warnings" +testpaths = [ + "tests", +] +filterwarnings = [ + "ignore::DeprecationWarning", +] + +[tool.coverage.run] +branch = true +source = ["src"] +omit = ["tests/*"] + +[tool.coverage.report] +show_missing = true +skip_covered = true +precision = 1 diff --git a/requirements.txt b/requirements.txt index a74ac47..ae9f222 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ pymupdf pillow -pytesseract \ No newline at end of file +pytesseract +pytest +coverage +flake8 +black \ No newline at end of file diff --git a/src/preprocessing/pdf_text_extraction.py b/src/preprocessing/pdf_text_extraction.py index c2d2ff0..8325035 100644 --- a/src/preprocessing/pdf_text_extraction.py +++ b/src/preprocessing/pdf_text_extraction.py @@ -17,6 +17,7 @@ from pathlib import Path import sys + def extract_text_from_pdf(pdf_path: str) -> str: text = [] try: @@ -28,13 +29,13 @@ def extract_text_from_pdf(pdf_path: str) -> str: # Clean out null bytes or UTF-16 artifacts if "\x00" in page_text: page_text = page_text.replace("\x00", "") - + # If the page is mostly empty, treat as image and use OCR if not page_text.strip(): pix = page.get_pixmap(dpi=300) img = Image.open(io.BytesIO(pix.tobytes("png"))) page_text = pytesseract.image_to_string(img) - + text.append(page_text) except Exception as e: print(f"[ERROR] Failed to extract text from {pdf_path}: {e}", file=sys.stderr) From bfefb18f9e7594b3b301e0c1640d169ba48a6da0 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 20:03:09 -0800 Subject: [PATCH 2/8] Created independent flake8 config file. --- .flake8 | 13 +++++ .github/workflows/working_sw.yml | 10 ++-- pyproject.toml | 8 +-- tests/test_pdf_extraction.py | 91 +++++++++++++++++++++++++++++++- 4 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..21bd171 --- /dev/null +++ b/.flake8 @@ -0,0 +1,13 @@ +# .flake8 +[flake8] +max-line-length = 200 +ignore = E203,W503 +exclude = + .git, + __pycache__, + build, + dist, + venv, + .venv, + .mypy_cache, + htmlcov diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 15a77f1..1d6fec2 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -24,16 +24,16 @@ jobs: pip install black flake8 coverage pytest # Code quality checks - - name: Run linter (Flake8) - run: | - echo "Running flake8 linter..." - flake8 src/ tests/ - - name: Check code formatting (Black) run: | echo "Checking code format with Black..." black --check src tests + - name: Run linter (Flake8) + run: | + echo "Running flake8 linter..." + flake8 src/ tests/ + # Functional validation - name: Run PDF extraction script run: | diff --git a/pyproject.toml b/pyproject.toml index d89789b..f51f9e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,16 +3,10 @@ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [tool.black] -line-length = 100 +line-length = 200 target-version = ["py311"] skip-string-normalization = true -[tool.flake8] -max-line-length = 100 -ignore = [ - "E203", # whitespace before ':' - "W503", # line break before binary operator -] exclude = [ ".git", "__pycache__", diff --git a/tests/test_pdf_extraction.py b/tests/test_pdf_extraction.py index 2c12727..d9b03b9 100644 --- a/tests/test_pdf_extraction.py +++ b/tests/test_pdf_extraction.py @@ -1,8 +1,95 @@ -from src.preprocessing.pdf_text_extraction import extract_text_from_pdf +import pytest +import subprocess +import fitz from pathlib import Path +import sys +from src.preprocessing.pdf_text_extraction import ( + extract_text_from_pdf, + save_to_file, + main +) + def test_extract_text_exists(): pdf_path = Path("tests/test.pdf") text = extract_text_from_pdf(str(pdf_path)) assert text is not None - assert len(text) > 0 \ No newline at end of file + assert len(text) > 0 + + +def test_save_to_file_creates_output(tmp_path): + output_path = tmp_path / "output.txt" + text = "Sample text" + save_to_file(text, output_path) + assert output_path.exists() + assert output_path.read_text(encoding="utf-8") == text + + +def test_main_handles_missing_pdf(monkeypatch, tmp_path): + fake_pdf = tmp_path / "nonexistent.pdf" + test_args = ["pdf_to_text.py", str(fake_pdf)] + monkeypatch.setattr(sys, "argv", test_args) + with pytest.raises(SystemExit): + main() + + +def test_extract_text_invalid_path(tmp_path): + invalid_pdf = tmp_path / "does_not_exist.pdf" + result = extract_text_from_pdf(str(invalid_pdf)) + assert result == "" + + +def test_extract_text_with_ocr(tmp_path): + pdf_path = tmp_path / "image_only.pdf" + + # Create image-only PDF + doc = fitz.open() + page = doc.new_page() + # Draw white rectangle (no text layer) + rect = fitz.Rect(0, 0, 100, 100) + page.insert_textbox(rect, "") # empty text + pix = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 100, 100)) + page.insert_image(rect, pixmap=pix) + doc.save(pdf_path) + doc.close() + + text = extract_text_from_pdf(str(pdf_path)) + assert isinstance(text, str) + # Empty string is acceptable if OCR yields nothing, but must not crash + assert text is not None + + +def test_save_to_file(tmp_path): + output_file = tmp_path / "output.txt" + save_to_file("hello world", output_file) + assert output_file.exists() + contents = output_file.read_text() + assert "hello" in contents + + +def test_save_to_file_permission_error(monkeypatch, tmp_path): + output_file = tmp_path / "readonly.txt" + output_file.touch() + output_file.chmod(0o400) + + # Monkeypatch print to avoid cluttering stderr + monkeypatch.setattr("sys.stderr", open("/dev/null", "w")) + + # Should not raise exception even if write fails + save_to_file("data", output_file) + + +def test_main_cli(tmp_path): + input_pdf = "tests/test.pdf" + output_dir = Path("data/processed-text") + output_dir.mkdir(parents=True, exist_ok=True) + output_file = output_dir / "test.txt" + + result = subprocess.run( + ["python", "src/preprocessing/pdf_text_extraction.py", input_pdf], + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert output_file.exists() + assert output_file.stat().st_size > 0 From 6a68e5eed2a485b68715a1b2c0411e27c04f4ae8 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 20:07:20 -0800 Subject: [PATCH 3/8] Multi-line regex string for config file. --- pyproject.toml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f51f9e0..41ae39d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,17 +6,18 @@ build-backend = "setuptools.build_meta" line-length = 200 target-version = ["py311"] skip-string-normalization = true - -exclude = [ - ".git", - "__pycache__", - "build", - "dist", - "venv", - ".venv", - ".mypy_cache", - "htmlcov", -] +exclude = ''' +/( + \.git + | __pycache__ + | build + | dist + | venv + | \.venv + | \.mypy_cache + | htmlcov +)/ +''' [tool.pytest.ini_options] addopts = "-ra --disable-warnings" From a0aca2069382795db15e3790c1d83467c6c3936a Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 20:11:37 -0800 Subject: [PATCH 4/8] Consolidated config to pyproject.toml file. --- .flake8 | 13 ------------- pyproject.toml | 18 ++++++++++++++++-- tests/test_pdf_extraction.py | 6 +----- 3 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 21bd171..0000000 --- a/.flake8 +++ /dev/null @@ -1,13 +0,0 @@ -# .flake8 -[flake8] -max-line-length = 200 -ignore = E203,W503 -exclude = - .git, - __pycache__, - build, - dist, - venv, - .venv, - .mypy_cache, - htmlcov diff --git a/pyproject.toml b/pyproject.toml index 41ae39d..75ef02d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,13 +19,27 @@ exclude = ''' )/ ''' +[tool.flake8] +max-line-length = 200 +ignore = ["E501", "E203", "W503"] +exclude = [ + ".git", + "__pycache__", + "build", + "dist", + "venv", + ".venv", + ".mypy_cache", + "htmlcov" +] + [tool.pytest.ini_options] addopts = "-ra --disable-warnings" testpaths = [ - "tests", + "tests" ] filterwarnings = [ - "ignore::DeprecationWarning", + "ignore::DeprecationWarning" ] [tool.coverage.run] diff --git a/tests/test_pdf_extraction.py b/tests/test_pdf_extraction.py index d9b03b9..d59e1e7 100644 --- a/tests/test_pdf_extraction.py +++ b/tests/test_pdf_extraction.py @@ -3,11 +3,7 @@ import fitz from pathlib import Path import sys -from src.preprocessing.pdf_text_extraction import ( - extract_text_from_pdf, - save_to_file, - main -) +from src.preprocessing.pdf_text_extraction import extract_text_from_pdf, save_to_file, main def test_extract_text_exists(): From 2fe94332fb43f48f3baf2200388593e361038f5c Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 20:18:00 -0800 Subject: [PATCH 5/8] Formatting fix. --- src/preprocessing/pdf_text_extraction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/preprocessing/pdf_text_extraction.py b/src/preprocessing/pdf_text_extraction.py index 8325035..cb0295f 100644 --- a/src/preprocessing/pdf_text_extraction.py +++ b/src/preprocessing/pdf_text_extraction.py @@ -1,4 +1,4 @@ -""" PDF Text Extraction Script +"""PDF Text Extraction Script Usage: python pdf_to_text.py path/to/input.pdf From a5a550a7769f7a6507f0f6833fd0a5eb98552d71 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 20:22:37 -0800 Subject: [PATCH 6/8] Version pinning linter and formatter. --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ae9f222..b8e9e82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ pillow pytesseract pytest coverage -flake8 -black \ No newline at end of file +flake8==6.1.0 +black==25.9.0 \ No newline at end of file From d7c251ac9d583f627d26b3749ecc43b7923a0329 Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 20:28:24 -0800 Subject: [PATCH 7/8] Removed uneccessary dependency installation in CI. --- .github/workflows/working_sw.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 1d6fec2..bb2c90d 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -21,7 +21,6 @@ jobs: sudo apt-get install -y tesseract-ocr python -m pip install --upgrade pip pip install -r requirements.txt - pip install black flake8 coverage pytest # Code quality checks - name: Check code formatting (Black) From 1f516c745ee2e1d3c0258624451d187a22d2ba3b Mon Sep 17 00:00:00 2001 From: SeanClay10 Date: Sun, 2 Nov 2025 20:32:17 -0800 Subject: [PATCH 8/8] Reintroduce flake8 config file. --- .flake8 | 12 ++++++++++++ pyproject.toml | 14 -------------- 2 files changed, 12 insertions(+), 14 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..0bd1b1f --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +max-line-length = 200 +ignore = E501,E203,W503 +exclude = + .git, + __pycache__, + build, + dist, + venv, + .venv, + .mypy_cache, + htmlcov diff --git a/pyproject.toml b/pyproject.toml index 75ef02d..42da6f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,20 +19,6 @@ exclude = ''' )/ ''' -[tool.flake8] -max-line-length = 200 -ignore = ["E501", "E203", "W503"] -exclude = [ - ".git", - "__pycache__", - "build", - "dist", - "venv", - ".venv", - ".mypy_cache", - "htmlcov" -] - [tool.pytest.ini_options] addopts = "-ra --disable-warnings" testpaths = [