diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 76bb016..dbf8647 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -21,26 +21,27 @@ jobs: sudo apt-get install -y tesseract-ocr python -m pip install --upgrade pip pip install -r requirements.txt - # OPTIONAL: Uncomment below after adding GOOGLE_SERVICE_ACCOUNT_JSON secret - # (Service Account JSON contents) to repo settings OR loading via .env. - # This step would download PDFs from a shared Google Drive folder. - # - name: Download training PDFs (Google Drive) - # env: - # GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} - # run: | - # python scripts/download_from_drive.py - # + + # CI pipeline (streams 20 PDFs per class from Drive and trains model) + - name: CI pipeline (Drive stream) + env: + GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} + GOOGLE_DRIVE_ROOT_FOLDER_ID: ${{ secrets.GOOGLE_DRIVE_ROOT_FOLDER_ID }} + PYTHONUNBUFFERED: 1 + run: | + python -u scripts/ci_pipeline.py + # OPTIONAL: Model cache and artifacts (commented until enabled) - # - name: Cache model artifacts - # uses: actions/cache@v3 - # with: - # path: | - # src/model/models - # ~/.joblib - # ~/.sklearn - # key: ${{ runner.os }}-sklearn-model-${{ hashFiles('src/model/**/*.py') }} - # restore-keys: | - # ${{ runner.os }}-sklearn-model- + - name: Cache model artifacts + uses: actions/cache@v3 + with: + path: | + src/model/models + ~/.joblib + ~/.sklearn + key: ${{ runner.os }}-sklearn-model-${{ hashFiles('src/model/**/*.py') }} + restore-keys: | + ${{ runner.os }}-sklearn-model- # Code quality checks - name: Check code formatting (Black) @@ -55,24 +56,22 @@ jobs: echo "Flake8 check completed." # Data preprocessing pipeline - - name: Run PDF extraction script + # Optional: still run a single-file extraction test on repo test asset + - name: Run PDF extraction script (repo test asset) run: | - mkdir -p data/processed-text - mkdir -p data/useful - mkdir -p data/not-useful - mkdir -p data/needs-check INPUT_PDF="tests/test.pdf" python src/preprocessing/pdf_text_extraction.py "$INPUT_PDF" - - name: Generate labels - run: | - echo "Generating labels for extracted text..." - python src/preprocessing/generate_labels.py + # - name: Generate labels + # run: | + # echo "Generating labels for extracted text..." + # python src/preprocessing/generate_labels.py - - name: Load and preprocess data - run: | - echo "Loading and preprocessing dataset..." - python src/preprocessing/data_loader.py + # - name: Load and preprocess data + # run: | + # echo "Loading and preprocessing dataset..." + # python src/preprocessing/data_loader.py + # No full pipeline here; use full_training_pipeline.py on main or a scheduled workflow - name: Validate preprocessing pipeline run: | @@ -92,12 +91,7 @@ jobs: python -c "import json; f=open('data/labels.json'); json.load(f); f.close()" || exit 1 # Model workflow temporarily disabled (training & classification skipped) - # - name: Train model - # run: | - # python src/model/train_model.py - # - name: Test model classification - # run: | - # python src/model/pdf_classifier.py --pdf-path "tests/test.pdf" --model_dir "src/model/models" + # Skip model training in CI for speed; exercised in full-training job on main/schedule # Unit testing and coverage - name: Run unit tests with coverage @@ -114,13 +108,13 @@ jobs: path: htmlcov/ # OPTIONAL: Upload trained model artifacts (enable once model steps are active) - # - name: Upload trained model - # uses: actions/upload-artifact@v4 - # with: - # name: trained-model - # path: | - # src/model/models/pdf_classifier_model.pkl - # src/model/models/tfidf_vectorizer.pkl + - name: Upload trained model + uses: actions/upload-artifact@v4 + with: + name: trained-model + path: | + src/model/models/pdf_classifier_model.pkl + src/model/models/tfidf_vectorizer.pkl - name: Show pipeline summary run: | @@ -129,4 +123,5 @@ jobs: ls -l data/processed-text/ echo "2. Label Generation Results:" ls -l data/labels.json - echo "(Model steps skipped in this run)" + echo "3. Model Training Results:" + ls -l src/model/models/ diff --git a/documentation/CONTRIBUTING.md b/documentation/CONTRIBUTING.md index 9a15d90..7702319 100644 --- a/documentation/CONTRIBUTING.md +++ b/documentation/CONTRIBUTING.md @@ -27,9 +27,25 @@ All contributors must follow the Oregon State University Student Code of Conduct pip install -r requirements.txt ``` * ### Running the application -``` - python src/main.py -``` + * If you do have the [dataset](https://drive.google.com/drive/u/2/folders/1U3_-TmnXnuBPR9vukkyV-3ITsxPr-nfo) downloaded locally on your machine: + ``` + python scripts/full_pipeline.py --local + ``` + * Note: Your local dataset folder should be formatted like so: + ``` + folder + |-> useful + |-> pdfs + |-> ... + |-> not-useful + |-> pdfs + |-> ... + ``` + * If you do not have the dataset downloaded or cannot have it on your locally on your machine: + ``` + python scripts/full_pipeline.py --api + ``` + * Note: You will need access to the .env file * ### Enviorment Variables * Sensitive information such as API keys will be stored in a local .env file which will be excluded by .gitignore. * Never hardcode secrets diff --git a/requirements.txt b/requirements.txt index 02307fd..358c777 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,8 @@ flake8==6.1.0 black==25.9.0 scikit-learn dotenv -numpy \ No newline at end of file +google-auth +google-auth-oauthlib +google-auth-httplib2 +numpy +google-api-python-client diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py new file mode 100644 index 0000000..38670fd --- /dev/null +++ b/scripts/ci_pipeline.py @@ -0,0 +1,102 @@ +"""CI pipeline: stream a small, fixed sample from Google Drive and run extraction. + +Requirements: + - Env GOOGLE_SERVICE_ACCOUNT_JSON set to Service Account JSON string. + - Env GOOGLE_DRIVE_ROOT_FOLDER_ID set to the Drive folder containing 'useful' and 'not-useful'. + - Optional env CI_FILES_PER_CLASS (default 1). + +This script DOES NOT save PDFs locally. It streams bytes and writes extracted text +into data/processed-text/*.txt, and writes data/labels.json. No training. +""" + +from __future__ import annotations + +import os +import json +from pathlib import Path +from typing import Dict, List +import subprocess + +import sys +from pathlib import Path as _Path + +sys.path.append(str(_Path(__file__).resolve().parents[1])) # add repo root to sys.path + +from scripts.env_loader import load_env + +load_env() # Load .env file if present (for local dev) + +from scripts.google_drive.drive_io import ( + get_drive_service, + find_child_folder_id, + list_pdfs_in_folder, + download_file_bytes, + sanitize_filename, +) +from src.preprocessing.pdf_text_extraction import extract_text_from_pdf_bytes + + +def write_labels(labels: Dict[str, str], output_file: Path): + output_file.parent.mkdir(parents=True, exist_ok=True) + with output_file.open("w", encoding="utf-8") as f: + json.dump(labels, f, indent=2) + + +def main(): + root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") + if not root_id: + raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") + + per_class = 5 # Fixed set size for CI + + print("Connecting to Google Drive...") + service = get_drive_service() + print("Connected to Google Drive") + + print(f"Looking for 'useful' and 'not-useful' folders under root {root_id}...") + useful_id = find_child_folder_id(service, root_id, "useful") + not_useful_id = find_child_folder_id(service, root_id, "not-useful") + if not useful_id: + raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") + if not not_useful_id: + raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") + print(f"Found both folders (useful: {useful_id}, not-useful: {not_useful_id})") + + # Prepare output directory + out_dir = Path("data/processed-text") + out_dir.mkdir(parents=True, exist_ok=True) + print(f"Output directory ready: {out_dir}") + + labels: Dict[str, str] = {} + + for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not useful")]: + print(f"\nProcessing '{label}' folder (max {per_class} PDFs)...") + files = list_pdfs_in_folder(service, folder_id, max_files=per_class) + print(f"Found {len(files)} PDFs in '{label}' folder") + for idx, f in enumerate(files, 1): + pdf_name = f.get("name", f.get("id", "file")) + print(f"[{idx}/{len(files)}] Processing: {pdf_name}") + pdf_bytes = download_file_bytes(service, f["id"]) + print(f"Downloaded {len(pdf_bytes)} bytes") + text = extract_text_from_pdf_bytes(pdf_bytes) + stem = sanitize_filename(pdf_name) + txt_name = f"{stem}.txt" + (out_dir / txt_name).write_text(text, encoding="utf-8") + labels[txt_name] = label + print(f"Extracted {len(text)} chars to {txt_name}") + + write_labels(labels, Path("data/labels.json")) + print(f"\nWrote {len(labels)} labels to data/labels.json") + print(f"Extracted {len(labels)} text files to {out_dir}") + + # Train model on the CI sample + print("\nStarting model training on CI sample...") + r = subprocess.run([sys.executable, "src/model/train_model.py"], env={**os.environ, "CI_TRAIN": "1"}) + if r.returncode != 0: + print("Model training failed") + raise SystemExit(r.returncode) + print("Model training complete") + + +if __name__ == "__main__": + main() diff --git a/scripts/env_loader.py b/scripts/env_loader.py new file mode 100644 index 0000000..c88fff6 --- /dev/null +++ b/scripts/env_loader.py @@ -0,0 +1,21 @@ +"""Load environment variables from .env file for local development.""" + +from pathlib import Path + + +def load_env(): + """Load .env file from repo root if it exists.""" + env_file = Path(__file__).resolve().parents[1] / ".env" + if not env_file.exists(): + return # No .env file, assume env vars set manually or in CI + + import os + + with env_file.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" in line: + key, value = line.split("=", 1) + os.environ.setdefault(key.strip(), value.strip()) diff --git a/scripts/full_pipeline.py b/scripts/full_pipeline.py new file mode 100644 index 0000000..95c2de2 --- /dev/null +++ b/scripts/full_pipeline.py @@ -0,0 +1,179 @@ +"""Full training pipeline: stream ALL PDFs from Drive, extract text, label, train model. + +Modes: + - API mode: Download PDFs from Google Drive and process them + - Local mode: Use PDFs already downloaded locally + +API Mode Environment variables: + - GOOGLE_SERVICE_ACCOUNT_JSON (service account JSON string) + - GOOGLE_DRIVE_ROOT_FOLDER_ID (root folder containing 'useful' and 'not-useful') + - GOOGLE_DRIVE_USE_SHARED_DRIVE=true (if using shared drives / shared folders) + +Usage: + - API mode: python full_pipeline.py --api + - Local mode: python full_pipeline.py --local + +Behavior: + - API mode: Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. + - Local mode: Processes PDFs from specified local directory (expects 'useful' and 'not-useful' subfolders). + - Generates labels.json based on folder origin. + - Trains model with src/model/train_model.py. +""" + +from __future__ import annotations + +import os +import json +import argparse +from pathlib import Path +from typing import Dict +import subprocess +import sys + +import sys +from pathlib import Path as _Path2 + +sys.path.append(str(_Path2(__file__).resolve().parents[1])) # add repo root to sys.path + +from scripts.env_loader import load_env + +load_env() # Load .env file if present (for local dev) + +from scripts.google_drive.drive_io import ( + get_drive_service, + find_child_folder_id, + list_pdfs_in_folder, + download_file_bytes, + sanitize_filename, +) +from src.preprocessing.pdf_text_extraction import extract_text_from_pdf_bytes + + +def run(cmd): + print(f"$ {' '.join(cmd)}") + r = subprocess.run(cmd) + if r.returncode != 0: + sys.exit(r.returncode) + + +def write_labels(labels: Dict[str, str], output_file: Path): + output_file.parent.mkdir(parents=True, exist_ok=True) + with output_file.open("w", encoding="utf-8") as f: + json.dump(labels, f, indent=2) + + +def process_api_mode(): + """Download PDFs from Google Drive and process them.""" + root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") + if not root_id: + raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") + + service = get_drive_service() + useful_id = find_child_folder_id(service, root_id, "useful") + not_useful_id = find_child_folder_id(service, root_id, "not-useful") + if not useful_id: + raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") + if not not_useful_id: + raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") + + out_dir = Path("data/processed-text") + out_dir.mkdir(parents=True, exist_ok=True) + labels: Dict[str, str] = {} + count=1 + for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not-useful")]: + files = list_pdfs_in_folder(service, folder_id, max_files=None) + print(f"Found {len(files)} PDFs in folder label '{label}'") + for f in files: + pdf_bytes = download_file_bytes(service, f["id"]) + text = extract_text_from_pdf_bytes(pdf_bytes) + stem = sanitize_filename(f.get("name", f.get("id", "file"))) + txt_name = f"{stem}.txt" + (out_dir / txt_name).write_text(text, encoding="utf-8") + labels[txt_name] = label + print(f"{count} Processed {f['name']}") + count+=1 + + write_labels(labels, Path("data/labels.json")) + print(f"Wrote {len(labels)} labeled text files.") + + +def process_local_mode(data_path: Path): + """Process PDFs from local directory.""" + if not data_path.exists(): + raise RuntimeError(f"Data path does not exist: {data_path}") + + useful_dir = data_path / "useful" + not_useful_dir = data_path / "not-useful" + + if not useful_dir.exists(): + raise RuntimeError(f"'useful' subfolder not found in {data_path}") + if not not_useful_dir.exists(): + raise RuntimeError(f"'not-useful' subfolder not found in {data_path}") + + out_dir = Path("data/processed-text") + out_dir.mkdir(parents=True, exist_ok=True) + labels: Dict[str, str] = {} + + for folder, label in [(useful_dir, "useful"), (not_useful_dir, "not-useful")]: + pdf_files = list(folder.glob("*.pdf")) + print(f"Found {len(pdf_files)} PDFs in local folder '{label}'") + + for pdf_path in pdf_files: + try: + with open(pdf_path, "rb") as f: + pdf_bytes = f.read() + text = extract_text_from_pdf_bytes(pdf_bytes) + stem = pdf_path.stem + txt_name = f"{stem}.txt" + (out_dir / txt_name).write_text(text, encoding="utf-8") + labels[txt_name] = label + print(f"Processed {pdf_path.name}") + except Exception as e: + print(f"Error processing {pdf_path.name}: {e}") + continue + + write_labels(labels, Path("data/labels.json")) + print(f"Wrote {len(labels)} labeled text files.") + + +def main(): + parser = argparse.ArgumentParser( + description="Full pipeline: extract text from PDFs and train model", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + API mode: python full_pipeline.py --api + Local mode: python full_pipeline.py --local ./data/pdfs + """ + ) + + # Create mutually exclusive group for --api and --local + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--api", + action="store_true", + help="Use API mode to download PDFs from Google Drive" + ) + group.add_argument( + "--local", + type=Path, + metavar="PATH", + help="Use local mode with PDFs from specified directory (should contain 'useful' and 'not-useful' subfolders)" + ) + + args = parser.parse_args() + + if args.local: + print(f"Running in LOCAL mode with data path: {args.local}") + process_local_mode(args.local) + else: # args.api + print("Running in API mode (Google Drive)") + process_api_mode() + + print("Beginning model training...") + run([sys.executable, "src/model/train_model.py"]) + print("Training complete.") + + +if __name__ == "__main__": + main() diff --git a/scripts/google_drive/download_from_drive.py b/scripts/google_drive/download_from_drive.py new file mode 100644 index 0000000..b5bab26 --- /dev/null +++ b/scripts/google_drive/download_from_drive.py @@ -0,0 +1,5 @@ +import os, json, google.auth +from google.oauth2 import service_account + +creds_info = json.loads(os.environ["GOOGLE_SERVICE_ACCOUNT_JSON"]) +creds = service_account.Credentials.from_service_account_info(creds_info) diff --git a/scripts/google_drive/drive_io.py b/scripts/google_drive/drive_io.py new file mode 100644 index 0000000..a2aa88c --- /dev/null +++ b/scripts/google_drive/drive_io.py @@ -0,0 +1,111 @@ +"""Google Drive I/O utilities using a Service Account. + +Environment variables required: + - GOOGLE_SERVICE_ACCOUNT_JSON: JSON string of the service account credentials + - GOOGLE_DRIVE_ROOT_FOLDER_ID: ID of the root folder containing 'useful' and 'not-useful' + +Optional: + - GOOGLE_DRIVE_USE_SHARED_DRIVE=true (enables includeItemsFromAllDrives/supportsAllDrives) + +This module streams PDF bytes without saving the PDF to disk. +""" + +from __future__ import annotations + +import os +import io +import re +from typing import List, Dict, Optional, Tuple + +from google.oauth2 import service_account +from googleapiclient.discovery import build +from googleapiclient.http import MediaIoBaseDownload + +from scripts.env_loader import load_env + +SCOPES = ["https://www.googleapis.com/auth/drive.readonly"] + + +def _use_all_drives() -> bool: + return os.environ.get("GOOGLE_DRIVE_USE_SHARED_DRIVE", "false").lower() in {"1", "true", "yes"} + + +def get_drive_service(): + load_env() # Load .env file if present + creds_info = os.environ.get("GOOGLE_SERVICE_ACCOUNT_JSON") + if not creds_info: + raise RuntimeError("Missing GOOGLE_SERVICE_ACCOUNT_JSON environment variable") + + import json + + creds = service_account.Credentials.from_service_account_info(json.loads(creds_info), scopes=SCOPES) + return build("drive", "v3", credentials=creds, cache_discovery=False) + + +def find_child_folder_id(service, parent_id: str, name: str) -> Optional[str]: + q = f"mimeType = 'application/vnd.google-apps.folder' and name = '{name}' and '{parent_id}' in parents and trashed = false" + params = { + "q": q, + "fields": "files(id, name)", + "pageSize": 10, + } + if _use_all_drives(): + params.update( + { + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + } + ) + resp = service.files().list(**params).execute() + files = resp.get("files", []) + return files[0]["id"] if files else None + + +def list_pdfs_in_folder(service, folder_id: str, max_files: Optional[int] = None, order_desc: bool = True) -> List[Dict]: + q = f"'{folder_id}' in parents and mimeType = 'application/pdf' and trashed = false" + params = { + "q": q, + "fields": "nextPageToken, files(id, name, mimeType, modifiedTime)", + "pageSize": 100, + "orderBy": "modifiedTime desc" if order_desc else "modifiedTime", + } + if _use_all_drives(): + params.update( + { + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + } + ) + + results: List[Dict] = [] + while True: + resp = service.files().list(**params).execute() + results.extend(resp.get("files", [])) + if max_files and len(results) >= max_files: + return results[:max_files] + token = resp.get("nextPageToken") + if not token: + break + params["pageToken"] = token + return results + + +def download_file_bytes(service, file_id: str) -> bytes: + request = service.files().get_media(fileId=file_id, supportsAllDrives=_use_all_drives()) + buf = io.BytesIO() + downloader = MediaIoBaseDownload(buf, request, chunksize=1024 * 1024) + done = False + while not done: + status, done = downloader.next_chunk() + # Optionally, could print progress: status.progress() + return buf.getvalue() + + +def sanitize_filename(name: str) -> str: + # Remove extension for stem-like behavior + name = re.sub(r"\.[Pp][Dd][Ff]$", "", name) + # Replace unsafe chars with underscore + name = re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("._-") + if not name: + name = "file" + return name diff --git a/scripts/run_pipeline.py b/scripts/run_pipeline.py deleted file mode 100644 index 1e299a5..0000000 --- a/scripts/run_pipeline.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Run the end-to-end local pipeline: - - Extract text from PDFs in data/useful and data/not-useful - - Generate labels.json - - Optionally train the model (disabled by default) - -Usage (PowerShell): - python scripts/run_pipeline.py - python scripts/run_pipeline.py --train -""" - -import argparse -import subprocess -from pathlib import Path -import sys - - -def run(cmd: list[str]): - print(f"\n$ {' '.join(cmd)}") - result = subprocess.run(cmd) - if result.returncode != 0: - print(f"[ERROR] Command failed: {' '.join(cmd)}", file=sys.stderr) - sys.exit(result.returncode) - - -def extract_folder(pdf_dir: Path): - if not pdf_dir.exists(): - print(f"[WARN] Missing folder: {pdf_dir}") - return - for pdf in pdf_dir.glob("*.pdf"): - run([sys.executable, "src/preprocessing/pdf_text_extraction.py", str(pdf)]) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--train", action="store_true", help="Also train the model after preprocessing") - args = parser.parse_args() - - # Ensure directories - Path("data/processed-text").mkdir(parents=True, exist_ok=True) - - # Extract text from both classes - extract_folder(Path("data/useful")) - extract_folder(Path("data/not-useful")) - - # Generate labels - run([sys.executable, "src/preprocessing/generate_labels.py"]) - - if args.train: - run([sys.executable, "src/model/train_model.py"]) - - -if __name__ == "__main__": - main() diff --git a/src/model/train_model.py b/src/model/train_model.py index 9b298d6..ef79e24 100644 --- a/src/model/train_model.py +++ b/src/model/train_model.py @@ -17,6 +17,7 @@ import json import sys from collections import Counter +import os # Load processed text files and their useful vs not useful labels. @@ -56,29 +57,45 @@ def train_pdf_classifier(texts, labels, output_dir="src/model/models"): if any(c < 2 for c in class_counts.values()): print(f"[ERROR] Each class needs at least 2 samples for stratified split. Counts: {class_counts}") return None - # Split the dataset into training and testing sets (stratified by label ratio) + + # Decide split strategy accommodating very small CI samples. + X_train = X_test = y_train = y_test = None + perform_eval = True + min_class = min(class_counts.values()) + n = len(labels) + test_size = 0.2 + tiny_context = n < 10 or min_class < 3 or os.environ.get("CI_TRAIN", "").lower() in {"1", "true", "yes"} + if tiny_context: + # For tiny datasets, attempt a 50/50 split; if that fails, fall back to train-only. + test_size = 0.5 try: - X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=0.2, random_state=42, stratify=labels) + X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=test_size, random_state=42, stratify=labels) except ValueError as e: - print(f"[ERROR] train_test_split failed: {e}") - return None + print(f"[WARN] Stratified split failed on tiny dataset: {e} — training on all data without evaluation.") + perform_eval = False + X_train = texts + y_train = labels # Convert text data into TF-IDF features vectorizer = TfidfVectorizer(max_features=10000, stop_words="english", ngram_range=(1, 3)) X_train_vec = vectorizer.fit_transform(X_train) - X_test_vec = vectorizer.transform(X_test) + X_test_vec = vectorizer.transform(X_test) if perform_eval else None # Initialize a logistic regression classifier model = LogisticRegression(max_iter=100000, solver="liblinear") model.fit(X_train_vec, y_train) # Train the model # Evaluate model performance - y_pred = model.predict(X_test_vec) - acc = accuracy_score(y_test, y_pred) - print("\n=== Model Evaluation ===") - print(f"Accuracy: {acc:.2f}") - print(classification_report(y_test, y_pred, digits=2)) - print("========================\n") + if perform_eval and X_test_vec is not None: + y_pred = model.predict(X_test_vec) + acc = accuracy_score(y_test, y_pred) + print("\n=== Model Evaluation ===") + print(f"Accuracy: {acc:.2f}") + print(classification_report(y_test, y_pred, digits=2)) + print("========================\n") + else: + acc = None + print("[INFO] Trained on full tiny dataset (no evaluation split).") # Save model and vectorizer for future use Path(output_dir).mkdir(parents=True, exist_ok=True) diff --git a/src/preprocessing/pdf_text_extraction.py b/src/preprocessing/pdf_text_extraction.py index 10ab0d3..bbd9a6b 100644 --- a/src/preprocessing/pdf_text_extraction.py +++ b/src/preprocessing/pdf_text_extraction.py @@ -48,6 +48,26 @@ def extract_text_from_pdf(pdf_path: str) -> str: return "\n".join(text) +def extract_text_from_pdf_bytes(data: bytes) -> str: + """Extract text from an in-memory PDF without writing the PDF to disk.""" + text = [] + try: + with fitz.open(stream=data, filetype="pdf") as doc: + for page_num, page in enumerate(doc, start=1): + page_text = page.get_text("text") + if "\x00" in page_text: + page_text = page_text.replace("\x00", "") + if not page_text.strip(): + pix = page.get_pixmap(dpi=300) + img = Image.open(io.BytesIO(pix.tobytes("png"))) + page_text = pytesseract.image_to_string(img) + text.append(page_text) + except Exception as e: + print(f"[ERROR] Failed to extract text from PDF bytes: {e}", file=sys.stderr) + return "" + return "\n".join(text) + + # Save extracted text to a file. def save_to_file(text: str, output_path: str): try: