From d86dab1d868af933b1005d4b082073e1ff77a708 Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 12:24:50 -0800 Subject: [PATCH 01/15] feat(ci): added google auth api key --- .github/workflows/working_sw.yml | 58 ++++++++++++++++---------------- requirements.txt | 5 ++- scripts/download_from_drive.py | 5 +++ 3 files changed, 38 insertions(+), 30 deletions(-) create mode 100644 scripts/download_from_drive.py diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 76bb016..e59aa3a 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -24,23 +24,23 @@ jobs: # OPTIONAL: Uncomment below after adding GOOGLE_SERVICE_ACCOUNT_JSON secret # (Service Account JSON contents) to repo settings OR loading via .env. # This step would download PDFs from a shared Google Drive folder. - # - name: Download training PDFs (Google Drive) - # env: - # GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} - # run: | - # python scripts/download_from_drive.py - # + - name: Download training PDFs (Google Drive) + env: + GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} + run: | + python scripts/download_from_drive.py + # OPTIONAL: Model cache and artifacts (commented until enabled) - # - name: Cache model artifacts - # uses: actions/cache@v3 - # with: - # path: | - # src/model/models - # ~/.joblib - # ~/.sklearn - # key: ${{ runner.os }}-sklearn-model-${{ hashFiles('src/model/**/*.py') }} - # restore-keys: | - # ${{ runner.os }}-sklearn-model- + - name: Cache model artifacts + uses: actions/cache@v3 + with: + path: | + src/model/models + ~/.joblib + ~/.sklearn + key: ${{ runner.os }}-sklearn-model-${{ hashFiles('src/model/**/*.py') }} + restore-keys: | + ${{ runner.os }}-sklearn-model- # Code quality checks - name: Check code formatting (Black) @@ -92,12 +92,12 @@ jobs: python -c "import json; f=open('data/labels.json'); json.load(f); f.close()" || exit 1 # Model workflow temporarily disabled (training & classification skipped) - # - name: Train model - # run: | - # python src/model/train_model.py - # - name: Test model classification - # run: | - # python src/model/pdf_classifier.py --pdf-path "tests/test.pdf" --model_dir "src/model/models" + - name: Train model + run: | + python src/model/train_model.py + - name: Test model classification + run: | + python src/model/pdf_classifier.py --pdf-path "tests/test.pdf" --model_dir "src/model/models" # Unit testing and coverage - name: Run unit tests with coverage @@ -114,13 +114,13 @@ jobs: path: htmlcov/ # OPTIONAL: Upload trained model artifacts (enable once model steps are active) - # - name: Upload trained model - # uses: actions/upload-artifact@v4 - # with: - # name: trained-model - # path: | - # src/model/models/pdf_classifier_model.pkl - # src/model/models/tfidf_vectorizer.pkl + - name: Upload trained model + uses: actions/upload-artifact@v4 + with: + name: trained-model + path: | + src/model/models/pdf_classifier_model.pkl + src/model/models/tfidf_vectorizer.pkl - name: Show pipeline summary run: | diff --git a/requirements.txt b/requirements.txt index 99f246d..038be0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,7 @@ coverage flake8==6.1.0 black==25.9.0 scikit-learn -dotenv \ No newline at end of file +dotenv +google-auth +google-auth-oauthlib +google-auth-httplib2 \ No newline at end of file diff --git a/scripts/download_from_drive.py b/scripts/download_from_drive.py new file mode 100644 index 0000000..bacbb26 --- /dev/null +++ b/scripts/download_from_drive.py @@ -0,0 +1,5 @@ +import os, json, google.auth +from google.oauth2 import service_account + +creds_info = json.loads(os.environ["GOOGLE_SERVICE_ACCOUNT_JSON"]) +creds = service_account.Credentials.from_service_account_info(creds_info) \ No newline at end of file From 35b686849b5fd7be9754e2389a91fbba2a05537a Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 12:35:17 -0800 Subject: [PATCH 02/15] fix(ci): use run_pipeline instead of calling each file manually --- .github/workflows/working_sw.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index e59aa3a..1ca461c 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -57,22 +57,25 @@ jobs: # Data preprocessing pipeline - name: Run PDF extraction script run: | - mkdir -p data/processed-text - mkdir -p data/useful - mkdir -p data/not-useful - mkdir -p data/needs-check + # mkdir -p data/processed-text + # mkdir -p data/useful + # mkdir -p data/not-useful + # mkdir -p data/needs-check INPUT_PDF="tests/test.pdf" python src/preprocessing/pdf_text_extraction.py "$INPUT_PDF" - - name: Generate labels - run: | - echo "Generating labels for extracted text..." - python src/preprocessing/generate_labels.py + # - name: Generate labels + # run: | + # echo "Generating labels for extracted text..." + # python src/preprocessing/generate_labels.py - - name: Load and preprocess data - run: | - echo "Loading and preprocessing dataset..." - python src/preprocessing/data_loader.py + # - name: Load and preprocess data + # run: | + # echo "Loading and preprocessing dataset..." + # python src/preprocessing/data_loader.py + - name: Run full pipeline + run: | + python scripts/run_pipeline.py - name: Validate preprocessing pipeline run: | From a9f509e4beca657e3c901ddc41ea91f9c96d1d5c Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:13:13 -0800 Subject: [PATCH 03/15] fix(ci): now fetches properly from gogole api --- .github/workflows/working_sw.yml | 31 +++---- requirements.txt | 1 + scripts/ci_pipeline.py | 85 ++++++++++++++++++ scripts/drive_io.py | 107 +++++++++++++++++++++++ scripts/env_loader.py | 20 +++++ src/model/train_model.py | 41 ++++++--- src/preprocessing/pdf_text_extraction.py | 20 +++++ 7 files changed, 274 insertions(+), 31 deletions(-) create mode 100644 scripts/ci_pipeline.py create mode 100644 scripts/drive_io.py create mode 100644 scripts/env_loader.py diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 1ca461c..7e6f14f 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -21,14 +21,14 @@ jobs: sudo apt-get install -y tesseract-ocr python -m pip install --upgrade pip pip install -r requirements.txt - # OPTIONAL: Uncomment below after adding GOOGLE_SERVICE_ACCOUNT_JSON secret - # (Service Account JSON contents) to repo settings OR loading via .env. - # This step would download PDFs from a shared Google Drive folder. - - name: Download training PDFs (Google Drive) + + # CI pipeline (streams 20 PDFs per class from Drive and trains model) + - name: CI pipeline (Drive stream) env: GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} + GOOGLE_DRIVE_ROOT_FOLDER_ID: ${{ secrets.GOOGLE_DRIVE_ROOT_FOLDER_ID }} run: | - python scripts/download_from_drive.py + python scripts/ci_pipeline.py # OPTIONAL: Model cache and artifacts (commented until enabled) - name: Cache model artifacts @@ -55,12 +55,9 @@ jobs: echo "Flake8 check completed." # Data preprocessing pipeline - - name: Run PDF extraction script + # Optional: still run a single-file extraction test on repo test asset + - name: Run PDF extraction script (repo test asset) run: | - # mkdir -p data/processed-text - # mkdir -p data/useful - # mkdir -p data/not-useful - # mkdir -p data/needs-check INPUT_PDF="tests/test.pdf" python src/preprocessing/pdf_text_extraction.py "$INPUT_PDF" @@ -73,9 +70,7 @@ jobs: # run: | # echo "Loading and preprocessing dataset..." # python src/preprocessing/data_loader.py - - name: Run full pipeline - run: | - python scripts/run_pipeline.py + # No full pipeline here; use full_training_pipeline.py on main or a scheduled workflow - name: Validate preprocessing pipeline run: | @@ -95,12 +90,7 @@ jobs: python -c "import json; f=open('data/labels.json'); json.load(f); f.close()" || exit 1 # Model workflow temporarily disabled (training & classification skipped) - - name: Train model - run: | - python src/model/train_model.py - - name: Test model classification - run: | - python src/model/pdf_classifier.py --pdf-path "tests/test.pdf" --model_dir "src/model/models" + # Skip model training in CI for speed; exercised in full-training job on main/schedule # Unit testing and coverage - name: Run unit tests with coverage @@ -132,4 +122,5 @@ jobs: ls -l data/processed-text/ echo "2. Label Generation Results:" ls -l data/labels.json - echo "(Model steps skipped in this run)" + echo "3. Model Training Results:" + ls -l src/model/models/ diff --git a/requirements.txt b/requirements.txt index f6c55cb..358c777 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ google-auth google-auth-oauthlib google-auth-httplib2 numpy +google-api-python-client diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py new file mode 100644 index 0000000..99e2bc4 --- /dev/null +++ b/scripts/ci_pipeline.py @@ -0,0 +1,85 @@ +"""CI pipeline: stream a small, fixed sample from Google Drive and run extraction. + +Requirements: + - Env GOOGLE_SERVICE_ACCOUNT_JSON set to Service Account JSON string. + - Env GOOGLE_DRIVE_ROOT_FOLDER_ID set to the Drive folder containing 'useful' and 'not-useful'. + - Optional env CI_FILES_PER_CLASS (default 1). + +This script DOES NOT save PDFs locally. It streams bytes and writes extracted text +into data/processed-text/*.txt, and writes data/labels.json. No training. +""" + +from __future__ import annotations + +import os +import json +from pathlib import Path +from typing import Dict, List +import subprocess + +import sys +from pathlib import Path as _Path +sys.path.append(str(_Path(__file__).resolve().parents[1])) # add repo root to sys.path + +from scripts.env_loader import load_env +load_env() # Load .env file if present (for local dev) + +from scripts.drive_io import ( + get_drive_service, + find_child_folder_id, + list_pdfs_in_folder, + download_file_bytes, + sanitize_filename, +) +from src.preprocessing.pdf_text_extraction import extract_text_from_pdf_bytes + + +def write_labels(labels: Dict[str, str], output_file: Path): + output_file.parent.mkdir(parents=True, exist_ok=True) + with output_file.open("w", encoding="utf-8") as f: + json.dump(labels, f, indent=2) + + +def main(): + root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") + if not root_id: + raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") + + per_class = 20 # Fixed set size for CI + + service = get_drive_service() + useful_id = find_child_folder_id(service, root_id, "useful") + not_useful_id = find_child_folder_id(service, root_id, "not-useful") + if not useful_id: + raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") + if not not_useful_id: + raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") + + # Prepare output directory + out_dir = Path("data/processed-text") + out_dir.mkdir(parents=True, exist_ok=True) + + labels: Dict[str, str] = {} + + for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not useful")]: + files = list_pdfs_in_folder(service, folder_id, max_files=per_class) + for f in files: + pdf_bytes = download_file_bytes(service, f["id"]) + text = extract_text_from_pdf_bytes(pdf_bytes) + stem = sanitize_filename(f.get("name", f.get("id", "file"))) + txt_name = f"{stem}.txt" + (out_dir / txt_name).write_text(text, encoding="utf-8") + labels[txt_name] = label + + write_labels(labels, Path("data/labels.json")) + print(f"[CI] Wrote {len(labels)} labels and extracted text files to {out_dir}") + + # Train model on the CI sample + print("[CI] Training model on CI sample...") + r = subprocess.run([sys.executable, "src/model/train_model.py"], env={**os.environ, "CI_TRAIN": "1"}) + if r.returncode != 0: + raise SystemExit(r.returncode) + + +if __name__ == "__main__": + main() diff --git a/scripts/drive_io.py b/scripts/drive_io.py new file mode 100644 index 0000000..81074cb --- /dev/null +++ b/scripts/drive_io.py @@ -0,0 +1,107 @@ +"""Google Drive I/O utilities using a Service Account. + +Environment variables required: + - GOOGLE_SERVICE_ACCOUNT_JSON: JSON string of the service account credentials + - GOOGLE_DRIVE_ROOT_FOLDER_ID: ID of the root folder containing 'useful' and 'not-useful' + +Optional: + - GOOGLE_DRIVE_USE_SHARED_DRIVE=true (enables includeItemsFromAllDrives/supportsAllDrives) + +This module streams PDF bytes without saving the PDF to disk. +""" + +from __future__ import annotations + +import os +import io +import re +from typing import List, Dict, Optional, Tuple + +from google.oauth2 import service_account +from googleapiclient.discovery import build +from googleapiclient.http import MediaIoBaseDownload + +from scripts.env_loader import load_env + +SCOPES = ["https://www.googleapis.com/auth/drive.readonly"] + + +def _use_all_drives() -> bool: + return os.environ.get("GOOGLE_DRIVE_USE_SHARED_DRIVE", "false").lower() in {"1", "true", "yes"} + + +def get_drive_service(): + load_env() # Load .env file if present + creds_info = os.environ.get("GOOGLE_SERVICE_ACCOUNT_JSON") + if not creds_info: + raise RuntimeError("Missing GOOGLE_SERVICE_ACCOUNT_JSON environment variable") + + import json + + creds = service_account.Credentials.from_service_account_info(json.loads(creds_info), scopes=SCOPES) + return build("drive", "v3", credentials=creds, cache_discovery=False) + + +def find_child_folder_id(service, parent_id: str, name: str) -> Optional[str]: + q = f"mimeType = 'application/vnd.google-apps.folder' and name = '{name}' and '{parent_id}' in parents and trashed = false" + params = { + "q": q, + "fields": "files(id, name)", + "pageSize": 10, + } + if _use_all_drives(): + params.update({ + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + }) + resp = service.files().list(**params).execute() + files = resp.get("files", []) + return files[0]["id"] if files else None + + +def list_pdfs_in_folder(service, folder_id: str, max_files: Optional[int] = None, order_desc: bool = True) -> List[Dict]: + q = f"'{folder_id}' in parents and mimeType = 'application/pdf' and trashed = false" + params = { + "q": q, + "fields": "nextPageToken, files(id, name, mimeType, modifiedTime)", + "pageSize": 100, + "orderBy": "modifiedTime desc" if order_desc else "modifiedTime", + } + if _use_all_drives(): + params.update({ + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + }) + + results: List[Dict] = [] + while True: + resp = service.files().list(**params).execute() + results.extend(resp.get("files", [])) + if max_files and len(results) >= max_files: + return results[:max_files] + token = resp.get("nextPageToken") + if not token: + break + params["pageToken"] = token + return results + + +def download_file_bytes(service, file_id: str) -> bytes: + request = service.files().get_media(fileId=file_id, supportsAllDrives=_use_all_drives()) + buf = io.BytesIO() + downloader = MediaIoBaseDownload(buf, request, chunksize=1024 * 1024) + done = False + while not done: + status, done = downloader.next_chunk() + # Optionally, could print progress: status.progress() + return buf.getvalue() + + +def sanitize_filename(name: str) -> str: + # Remove extension for stem-like behavior + name = re.sub(r"\.[Pp][Dd][Ff]$", "", name) + # Replace unsafe chars with underscore + name = re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("._-") + if not name: + name = "file" + return name diff --git a/scripts/env_loader.py b/scripts/env_loader.py new file mode 100644 index 0000000..ea8eb44 --- /dev/null +++ b/scripts/env_loader.py @@ -0,0 +1,20 @@ +"""Load environment variables from .env file for local development.""" + +from pathlib import Path + + +def load_env(): + """Load .env file from repo root if it exists.""" + env_file = Path(__file__).resolve().parents[1] / ".env" + if not env_file.exists(): + return # No .env file, assume env vars set manually or in CI + + import os + with env_file.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" in line: + key, value = line.split("=", 1) + os.environ.setdefault(key.strip(), value.strip()) diff --git a/src/model/train_model.py b/src/model/train_model.py index 9b298d6..405f8ad 100644 --- a/src/model/train_model.py +++ b/src/model/train_model.py @@ -17,6 +17,7 @@ import json import sys from collections import Counter +import os # Load processed text files and their useful vs not useful labels. @@ -56,29 +57,47 @@ def train_pdf_classifier(texts, labels, output_dir="src/model/models"): if any(c < 2 for c in class_counts.values()): print(f"[ERROR] Each class needs at least 2 samples for stratified split. Counts: {class_counts}") return None - # Split the dataset into training and testing sets (stratified by label ratio) + + # Decide split strategy accommodating very small CI samples. + X_train = X_test = y_train = y_test = None + perform_eval = True + min_class = min(class_counts.values()) + n = len(labels) + test_size = 0.2 + tiny_context = n < 10 or min_class < 3 or os.environ.get("CI_TRAIN", "").lower() in {"1", "true", "yes"} + if tiny_context: + # For tiny datasets, attempt a 50/50 split; if that fails, fall back to train-only. + test_size = 0.5 try: - X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=0.2, random_state=42, stratify=labels) + X_train, X_test, y_train, y_test = train_test_split( + texts, labels, test_size=test_size, random_state=42, stratify=labels + ) except ValueError as e: - print(f"[ERROR] train_test_split failed: {e}") - return None + print(f"[WARN] Stratified split failed on tiny dataset: {e} — training on all data without evaluation.") + perform_eval = False + X_train = texts + y_train = labels # Convert text data into TF-IDF features vectorizer = TfidfVectorizer(max_features=10000, stop_words="english", ngram_range=(1, 3)) X_train_vec = vectorizer.fit_transform(X_train) - X_test_vec = vectorizer.transform(X_test) + X_test_vec = vectorizer.transform(X_test) if perform_eval else None # Initialize a logistic regression classifier model = LogisticRegression(max_iter=100000, solver="liblinear") model.fit(X_train_vec, y_train) # Train the model # Evaluate model performance - y_pred = model.predict(X_test_vec) - acc = accuracy_score(y_test, y_pred) - print("\n=== Model Evaluation ===") - print(f"Accuracy: {acc:.2f}") - print(classification_report(y_test, y_pred, digits=2)) - print("========================\n") + if perform_eval and X_test_vec is not None: + y_pred = model.predict(X_test_vec) + acc = accuracy_score(y_test, y_pred) + print("\n=== Model Evaluation ===") + print(f"Accuracy: {acc:.2f}") + print(classification_report(y_test, y_pred, digits=2)) + print("========================\n") + else: + acc = None + print("[INFO] Trained on full tiny dataset (no evaluation split).") # Save model and vectorizer for future use Path(output_dir).mkdir(parents=True, exist_ok=True) diff --git a/src/preprocessing/pdf_text_extraction.py b/src/preprocessing/pdf_text_extraction.py index 10ab0d3..bbd9a6b 100644 --- a/src/preprocessing/pdf_text_extraction.py +++ b/src/preprocessing/pdf_text_extraction.py @@ -48,6 +48,26 @@ def extract_text_from_pdf(pdf_path: str) -> str: return "\n".join(text) +def extract_text_from_pdf_bytes(data: bytes) -> str: + """Extract text from an in-memory PDF without writing the PDF to disk.""" + text = [] + try: + with fitz.open(stream=data, filetype="pdf") as doc: + for page_num, page in enumerate(doc, start=1): + page_text = page.get_text("text") + if "\x00" in page_text: + page_text = page_text.replace("\x00", "") + if not page_text.strip(): + pix = page.get_pixmap(dpi=300) + img = Image.open(io.BytesIO(pix.tobytes("png"))) + page_text = pytesseract.image_to_string(img) + text.append(page_text) + except Exception as e: + print(f"[ERROR] Failed to extract text from PDF bytes: {e}", file=sys.stderr) + return "" + return "\n".join(text) + + # Save extracted text to a file. def save_to_file(text: str, output_path: str): try: From e17b8a25441cdac14cbf09a985503836d8f82781 Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:23:44 -0800 Subject: [PATCH 04/15] feat(ci): added printed statements to look where ci is at with processing pdfs --- scripts/ci_pipeline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index 99e2bc4..3523faf 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -64,12 +64,15 @@ def main(): for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not useful")]: files = list_pdfs_in_folder(service, folder_id, max_files=per_class) for f in files: + pdf_name = f.get("name", f.get("id", "file")) + print(f"[CI] Processing: {pdf_name} ({label})") pdf_bytes = download_file_bytes(service, f["id"]) text = extract_text_from_pdf_bytes(pdf_bytes) - stem = sanitize_filename(f.get("name", f.get("id", "file"))) + stem = sanitize_filename(pdf_name) txt_name = f"{stem}.txt" (out_dir / txt_name).write_text(text, encoding="utf-8") labels[txt_name] = label + print(f"[CI] ✓ Extracted to {txt_name}") write_labels(labels, Path("data/labels.json")) print(f"[CI] Wrote {len(labels)} labels and extracted text files to {out_dir}") From 5ecd89e77b95081f103d76f4a9f4001f393bd265 Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:26:59 -0800 Subject: [PATCH 05/15] feat(ci): more print statement checkpoints --- scripts/ci_pipeline.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index 3523faf..a822aff 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -47,41 +47,53 @@ def main(): per_class = 20 # Fixed set size for CI + print("[CI] Connecting to Google Drive...") service = get_drive_service() + print("[CI] Connected to Google Drive") + + print(f"[CI] Looking for 'useful' and 'not-useful' folders under root {root_id}...") useful_id = find_child_folder_id(service, root_id, "useful") not_useful_id = find_child_folder_id(service, root_id, "not-useful") if not useful_id: raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") if not not_useful_id: raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") + print(f"[CI] Found both folders (useful: {useful_id}, not-useful: {not_useful_id})") # Prepare output directory out_dir = Path("data/processed-text") out_dir.mkdir(parents=True, exist_ok=True) + print(f"[CI] Output directory ready: {out_dir}") labels: Dict[str, str] = {} for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not useful")]: + print(f"\n[CI] Processing '{label}' folder (max {per_class} PDFs)...") files = list_pdfs_in_folder(service, folder_id, max_files=per_class) - for f in files: + print(f"[CI] Found {len(files)} PDFs in '{label}' folder") + for idx, f in enumerate(files, 1): pdf_name = f.get("name", f.get("id", "file")) - print(f"[CI] Processing: {pdf_name} ({label})") + print(f"[CI] [{idx}/{len(files)}] Processing: {pdf_name}") pdf_bytes = download_file_bytes(service, f["id"]) + print(f"[CI] Downloaded {len(pdf_bytes)} bytes") text = extract_text_from_pdf_bytes(pdf_bytes) stem = sanitize_filename(pdf_name) txt_name = f"{stem}.txt" (out_dir / txt_name).write_text(text, encoding="utf-8") labels[txt_name] = label - print(f"[CI] ✓ Extracted to {txt_name}") + print(f"[CI] ✓ Extracted {len(text)} chars to {txt_name}") write_labels(labels, Path("data/labels.json")) - print(f"[CI] Wrote {len(labels)} labels and extracted text files to {out_dir}") + print(f"\n[CI] ✓ Wrote {len(labels)} labels to data/labels.json") + print(f"[CI] ✓ Extracted {len(labels)} text files to {out_dir}") # Train model on the CI sample - print("[CI] Training model on CI sample...") + print("\n[CI] Starting model training on CI sample...") r = subprocess.run([sys.executable, "src/model/train_model.py"], env={**os.environ, "CI_TRAIN": "1"}) if r.returncode != 0: + print("[CI] ✗ Model training failed") raise SystemExit(r.returncode) + print("[CI] ✓ Model training complete") if __name__ == "__main__": From 6bdf0f7a3517e38d2f46e43bffbf58c9cae73162 Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:33:21 -0800 Subject: [PATCH 06/15] fix(ci): unbuffer python print statemets --- .github/workflows/working_sw.yml | 3 +- scripts/ci_pipeline.py | 17 ++++-- scripts/full_pipeline.py | 90 ++++++++++++++++++++++++++++++++ scripts/run_pipeline.py | 54 ------------------- 4 files changed, 104 insertions(+), 60 deletions(-) create mode 100644 scripts/full_pipeline.py delete mode 100644 scripts/run_pipeline.py diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 7e6f14f..dbf8647 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -27,8 +27,9 @@ jobs: env: GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} GOOGLE_DRIVE_ROOT_FOLDER_ID: ${{ secrets.GOOGLE_DRIVE_ROOT_FOLDER_ID }} + PYTHONUNBUFFERED: 1 run: | - python scripts/ci_pipeline.py + python -u scripts/ci_pipeline.py # OPTIONAL: Model cache and artifacts (commented until enabled) - name: Cache model artifacts diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index a822aff..99ac10c 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -41,29 +41,36 @@ def write_labels(labels: Dict[str, str], output_file: Path): def main(): + print("[CI] Starting CI pipeline...", flush=True) + print(f"[CI] Checking environment variables...", flush=True) + root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") if not root_id: raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") + print(f"[CI] ✓ GOOGLE_DRIVE_ROOT_FOLDER_ID is set", flush=True) + + has_creds = bool(os.environ.get("GOOGLE_SERVICE_ACCOUNT_JSON")) + print(f"[CI] ✓ GOOGLE_SERVICE_ACCOUNT_JSON is {'set' if has_creds else 'MISSING'}", flush=True) per_class = 20 # Fixed set size for CI - print("[CI] Connecting to Google Drive...") + print("[CI] Connecting to Google Drive...", flush=True) service = get_drive_service() - print("[CI] Connected to Google Drive") + print("[CI] ✓ Connected to Google Drive", flush=True) - print(f"[CI] Looking for 'useful' and 'not-useful' folders under root {root_id}...") + print(f"[CI] Looking for 'useful' and 'not-useful' folders under root {root_id}...", flush=True) useful_id = find_child_folder_id(service, root_id, "useful") not_useful_id = find_child_folder_id(service, root_id, "not-useful") if not useful_id: raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") if not not_useful_id: raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") - print(f"[CI] Found both folders (useful: {useful_id}, not-useful: {not_useful_id})") + print(f"[CI] ✓ Found both folders (useful: {useful_id}, not-useful: {not_useful_id})", flush=True) # Prepare output directory out_dir = Path("data/processed-text") out_dir.mkdir(parents=True, exist_ok=True) - print(f"[CI] Output directory ready: {out_dir}") + print(f"[CI] ✓ Output directory ready: {out_dir}", flush=True) labels: Dict[str, str] = {} diff --git a/scripts/full_pipeline.py b/scripts/full_pipeline.py new file mode 100644 index 0000000..74244f3 --- /dev/null +++ b/scripts/full_pipeline.py @@ -0,0 +1,90 @@ +"""Full training pipeline: stream ALL PDFs from Drive, extract text, label, train model. + +Environment variables: + - GOOGLE_SERVICE_ACCOUNT_JSON (service account JSON string) + - GOOGLE_DRIVE_ROOT_FOLDER_ID (root folder containing 'useful' and 'not-useful') + - GOOGLE_DRIVE_USE_SHARED_DRIVE=true (if using shared drives / shared folders) + +Behavior: + - Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. + - Generates labels.json based on folder origin. + - Trains model with src/model/train_model.py. +""" + +from __future__ import annotations + +import os +import json +from pathlib import Path +from typing import Dict +import subprocess +import sys + +import sys +from pathlib import Path as _Path2 +sys.path.append(str(_Path2(__file__).resolve().parents[1])) # add repo root to sys.path + +from scripts.env_loader import load_env +load_env() # Load .env file if present (for local dev) + +from scripts.drive_io import ( + get_drive_service, + find_child_folder_id, + list_pdfs_in_folder, + download_file_bytes, + sanitize_filename, +) +from src.preprocessing.pdf_text_extraction import extract_text_from_pdf_bytes + + +def run(cmd): + print(f"$ {' '.join(cmd)}") + r = subprocess.run(cmd) + if r.returncode != 0: + sys.exit(r.returncode) + + +def write_labels(labels: Dict[str, str], output_file: Path): + output_file.parent.mkdir(parents=True, exist_ok=True) + with output_file.open("w", encoding="utf-8") as f: + json.dump(labels, f, indent=2) + + +def main(): + root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") + if not root_id: + raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") + + service = get_drive_service() + useful_id = find_child_folder_id(service, root_id, "useful") + not_useful_id = find_child_folder_id(service, root_id, "not-useful") + if not useful_id: + raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") + if not not_useful_id: + raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") + + out_dir = Path("data/processed-text") + out_dir.mkdir(parents=True, exist_ok=True) + labels: Dict[str, str] = {} + + for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not-useful")]: + files = list_pdfs_in_folder(service, folder_id, max_files=None) + print(f"[FULL] Found {len(files)} PDFs in folder label '{label}'") + for f in files: + pdf_bytes = download_file_bytes(service, f["id"]) + text = extract_text_from_pdf_bytes(pdf_bytes) + stem = sanitize_filename(f.get("name", f.get("id", "file"))) + txt_name = f"{stem}.txt" + (out_dir / txt_name).write_text(text, encoding="utf-8") + labels[txt_name] = label + + write_labels(labels, Path("data/labels.json")) + print(f"[FULL] Wrote {len(labels)} labeled text files. Beginning model training...") + + # Train model + run([sys.executable, "src/model/train_model.py"]) + print("[FULL] Training complete.") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_pipeline.py b/scripts/run_pipeline.py deleted file mode 100644 index 1e299a5..0000000 --- a/scripts/run_pipeline.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Run the end-to-end local pipeline: - - Extract text from PDFs in data/useful and data/not-useful - - Generate labels.json - - Optionally train the model (disabled by default) - -Usage (PowerShell): - python scripts/run_pipeline.py - python scripts/run_pipeline.py --train -""" - -import argparse -import subprocess -from pathlib import Path -import sys - - -def run(cmd: list[str]): - print(f"\n$ {' '.join(cmd)}") - result = subprocess.run(cmd) - if result.returncode != 0: - print(f"[ERROR] Command failed: {' '.join(cmd)}", file=sys.stderr) - sys.exit(result.returncode) - - -def extract_folder(pdf_dir: Path): - if not pdf_dir.exists(): - print(f"[WARN] Missing folder: {pdf_dir}") - return - for pdf in pdf_dir.glob("*.pdf"): - run([sys.executable, "src/preprocessing/pdf_text_extraction.py", str(pdf)]) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--train", action="store_true", help="Also train the model after preprocessing") - args = parser.parse_args() - - # Ensure directories - Path("data/processed-text").mkdir(parents=True, exist_ok=True) - - # Extract text from both classes - extract_folder(Path("data/useful")) - extract_folder(Path("data/not-useful")) - - # Generate labels - run([sys.executable, "src/preprocessing/generate_labels.py"]) - - if args.train: - run([sys.executable, "src/model/train_model.py"]) - - -if __name__ == "__main__": - main() From 087f0efdf912b9976e76f4fec3cf86d2c24bd305 Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:39:32 -0800 Subject: [PATCH 07/15] Revert "fix(ci): unbuffer python print statemets" This reverts commit 6bdf0f7a3517e38d2f46e43bffbf58c9cae73162. --- .github/workflows/working_sw.yml | 3 +-- scripts/ci_pipeline.py | 17 +++++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index dbf8647..7e6f14f 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -27,9 +27,8 @@ jobs: env: GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} GOOGLE_DRIVE_ROOT_FOLDER_ID: ${{ secrets.GOOGLE_DRIVE_ROOT_FOLDER_ID }} - PYTHONUNBUFFERED: 1 run: | - python -u scripts/ci_pipeline.py + python scripts/ci_pipeline.py # OPTIONAL: Model cache and artifacts (commented until enabled) - name: Cache model artifacts diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index 99ac10c..a822aff 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -41,36 +41,29 @@ def write_labels(labels: Dict[str, str], output_file: Path): def main(): - print("[CI] Starting CI pipeline...", flush=True) - print(f"[CI] Checking environment variables...", flush=True) - root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") if not root_id: raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") - print(f"[CI] ✓ GOOGLE_DRIVE_ROOT_FOLDER_ID is set", flush=True) - - has_creds = bool(os.environ.get("GOOGLE_SERVICE_ACCOUNT_JSON")) - print(f"[CI] ✓ GOOGLE_SERVICE_ACCOUNT_JSON is {'set' if has_creds else 'MISSING'}", flush=True) per_class = 20 # Fixed set size for CI - print("[CI] Connecting to Google Drive...", flush=True) + print("[CI] Connecting to Google Drive...") service = get_drive_service() - print("[CI] ✓ Connected to Google Drive", flush=True) + print("[CI] Connected to Google Drive") - print(f"[CI] Looking for 'useful' and 'not-useful' folders under root {root_id}...", flush=True) + print(f"[CI] Looking for 'useful' and 'not-useful' folders under root {root_id}...") useful_id = find_child_folder_id(service, root_id, "useful") not_useful_id = find_child_folder_id(service, root_id, "not-useful") if not useful_id: raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") if not not_useful_id: raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") - print(f"[CI] ✓ Found both folders (useful: {useful_id}, not-useful: {not_useful_id})", flush=True) + print(f"[CI] Found both folders (useful: {useful_id}, not-useful: {not_useful_id})") # Prepare output directory out_dir = Path("data/processed-text") out_dir.mkdir(parents=True, exist_ok=True) - print(f"[CI] ✓ Output directory ready: {out_dir}", flush=True) + print(f"[CI] Output directory ready: {out_dir}") labels: Dict[str, str] = {} From fff13b62459d55f98c20ce9038ebecd4a4c07f26 Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:41:57 -0800 Subject: [PATCH 08/15] fix(ci): unbuffer python print statemets --- .github/workflows/working_sw.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/working_sw.yml b/.github/workflows/working_sw.yml index 7e6f14f..dbf8647 100644 --- a/.github/workflows/working_sw.yml +++ b/.github/workflows/working_sw.yml @@ -27,8 +27,9 @@ jobs: env: GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }} GOOGLE_DRIVE_ROOT_FOLDER_ID: ${{ secrets.GOOGLE_DRIVE_ROOT_FOLDER_ID }} + PYTHONUNBUFFERED: 1 run: | - python scripts/ci_pipeline.py + python -u scripts/ci_pipeline.py # OPTIONAL: Model cache and artifacts (commented until enabled) - name: Cache model artifacts From 7cfd487b4e85d4587731ecd1ff578d3d7d6f2196 Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:44:37 -0800 Subject: [PATCH 09/15] fix(ci): lowered amount of papers used --- scripts/ci_pipeline.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index a822aff..56e244d 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -45,55 +45,55 @@ def main(): if not root_id: raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") - per_class = 20 # Fixed set size for CI + per_class = 5 # Fixed set size for CI - print("[CI] Connecting to Google Drive...") + print("Connecting to Google Drive...") service = get_drive_service() - print("[CI] Connected to Google Drive") + print("Connected to Google Drive") - print(f"[CI] Looking for 'useful' and 'not-useful' folders under root {root_id}...") + print(f"Looking for 'useful' and 'not-useful' folders under root {root_id}...") useful_id = find_child_folder_id(service, root_id, "useful") not_useful_id = find_child_folder_id(service, root_id, "not-useful") if not useful_id: raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}") if not not_useful_id: raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}") - print(f"[CI] Found both folders (useful: {useful_id}, not-useful: {not_useful_id})") + print(f"Found both folders (useful: {useful_id}, not-useful: {not_useful_id})") # Prepare output directory out_dir = Path("data/processed-text") out_dir.mkdir(parents=True, exist_ok=True) - print(f"[CI] Output directory ready: {out_dir}") + print(f"Output directory ready: {out_dir}") labels: Dict[str, str] = {} for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not useful")]: - print(f"\n[CI] Processing '{label}' folder (max {per_class} PDFs)...") + print(f"\nProcessing '{label}' folder (max {per_class} PDFs)...") files = list_pdfs_in_folder(service, folder_id, max_files=per_class) - print(f"[CI] Found {len(files)} PDFs in '{label}' folder") + print(f"Found {len(files)} PDFs in '{label}' folder") for idx, f in enumerate(files, 1): pdf_name = f.get("name", f.get("id", "file")) - print(f"[CI] [{idx}/{len(files)}] Processing: {pdf_name}") + print(f"[{idx}/{len(files)}] Processing: {pdf_name}") pdf_bytes = download_file_bytes(service, f["id"]) - print(f"[CI] Downloaded {len(pdf_bytes)} bytes") + print(f"Downloaded {len(pdf_bytes)} bytes") text = extract_text_from_pdf_bytes(pdf_bytes) stem = sanitize_filename(pdf_name) txt_name = f"{stem}.txt" (out_dir / txt_name).write_text(text, encoding="utf-8") labels[txt_name] = label - print(f"[CI] ✓ Extracted {len(text)} chars to {txt_name}") + print(f"Extracted {len(text)} chars to {txt_name}") write_labels(labels, Path("data/labels.json")) - print(f"\n[CI] ✓ Wrote {len(labels)} labels to data/labels.json") - print(f"[CI] ✓ Extracted {len(labels)} text files to {out_dir}") + print(f"\nWrote {len(labels)} labels to data/labels.json") + print(f"Extracted {len(labels)} text files to {out_dir}") # Train model on the CI sample - print("\n[CI] Starting model training on CI sample...") + print("\nStarting model training on CI sample...") r = subprocess.run([sys.executable, "src/model/train_model.py"], env={**os.environ, "CI_TRAIN": "1"}) if r.returncode != 0: - print("[CI] ✗ Model training failed") + print("Model training failed") raise SystemExit(r.returncode) - print("[CI] ✓ Model training complete") + print("Model training complete") if __name__ == "__main__": From 0219e84dfa564105788a3fbdc94b7e669509cbbc Mon Sep 17 00:00:00 2001 From: Raymond Cen Date: Wed, 12 Nov 2025 14:47:13 -0800 Subject: [PATCH 10/15] forat(black): reformatted code --- scripts/ci_pipeline.py | 4 +++- scripts/download_from_drive.py | 2 +- scripts/drive_io.py | 20 ++++++++++++-------- scripts/env_loader.py | 3 ++- scripts/full_pipeline.py | 2 ++ src/model/train_model.py | 4 +--- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index 56e244d..4eb6579 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -19,9 +19,11 @@ import sys from pathlib import Path as _Path + sys.path.append(str(_Path(__file__).resolve().parents[1])) # add repo root to sys.path from scripts.env_loader import load_env + load_env() # Load .env file if present (for local dev) from scripts.drive_io import ( @@ -50,7 +52,7 @@ def main(): print("Connecting to Google Drive...") service = get_drive_service() print("Connected to Google Drive") - + print(f"Looking for 'useful' and 'not-useful' folders under root {root_id}...") useful_id = find_child_folder_id(service, root_id, "useful") not_useful_id = find_child_folder_id(service, root_id, "not-useful") diff --git a/scripts/download_from_drive.py b/scripts/download_from_drive.py index bacbb26..b5bab26 100644 --- a/scripts/download_from_drive.py +++ b/scripts/download_from_drive.py @@ -2,4 +2,4 @@ from google.oauth2 import service_account creds_info = json.loads(os.environ["GOOGLE_SERVICE_ACCOUNT_JSON"]) -creds = service_account.Credentials.from_service_account_info(creds_info) \ No newline at end of file +creds = service_account.Credentials.from_service_account_info(creds_info) diff --git a/scripts/drive_io.py b/scripts/drive_io.py index 81074cb..a2aa88c 100644 --- a/scripts/drive_io.py +++ b/scripts/drive_io.py @@ -50,10 +50,12 @@ def find_child_folder_id(service, parent_id: str, name: str) -> Optional[str]: "pageSize": 10, } if _use_all_drives(): - params.update({ - "supportsAllDrives": True, - "includeItemsFromAllDrives": True, - }) + params.update( + { + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + } + ) resp = service.files().list(**params).execute() files = resp.get("files", []) return files[0]["id"] if files else None @@ -68,10 +70,12 @@ def list_pdfs_in_folder(service, folder_id: str, max_files: Optional[int] = None "orderBy": "modifiedTime desc" if order_desc else "modifiedTime", } if _use_all_drives(): - params.update({ - "supportsAllDrives": True, - "includeItemsFromAllDrives": True, - }) + params.update( + { + "supportsAllDrives": True, + "includeItemsFromAllDrives": True, + } + ) results: List[Dict] = [] while True: diff --git a/scripts/env_loader.py b/scripts/env_loader.py index ea8eb44..c88fff6 100644 --- a/scripts/env_loader.py +++ b/scripts/env_loader.py @@ -8,8 +8,9 @@ def load_env(): env_file = Path(__file__).resolve().parents[1] / ".env" if not env_file.exists(): return # No .env file, assume env vars set manually or in CI - + import os + with env_file.open("r", encoding="utf-8") as f: for line in f: line = line.strip() diff --git a/scripts/full_pipeline.py b/scripts/full_pipeline.py index 74244f3..ed2a94d 100644 --- a/scripts/full_pipeline.py +++ b/scripts/full_pipeline.py @@ -22,9 +22,11 @@ import sys from pathlib import Path as _Path2 + sys.path.append(str(_Path2(__file__).resolve().parents[1])) # add repo root to sys.path from scripts.env_loader import load_env + load_env() # Load .env file if present (for local dev) from scripts.drive_io import ( diff --git a/src/model/train_model.py b/src/model/train_model.py index 405f8ad..ef79e24 100644 --- a/src/model/train_model.py +++ b/src/model/train_model.py @@ -69,9 +69,7 @@ def train_pdf_classifier(texts, labels, output_dir="src/model/models"): # For tiny datasets, attempt a 50/50 split; if that fails, fall back to train-only. test_size = 0.5 try: - X_train, X_test, y_train, y_test = train_test_split( - texts, labels, test_size=test_size, random_state=42, stratify=labels - ) + X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=test_size, random_state=42, stratify=labels) except ValueError as e: print(f"[WARN] Stratified split failed on tiny dataset: {e} — training on all data without evaluation.") perform_eval = False From d482a60068be9d5b24b5a264da63b2c5bcb88151 Mon Sep 17 00:00:00 2001 From: raymondcen Date: Thu, 13 Nov 2025 13:51:35 -0800 Subject: [PATCH 11/15] feat(model pipeline): modified model pipeline to include api mode --- scripts/ci_pipeline.py | 2 +- scripts/full_pipeline.py | 105 ++++++++++++++++-- .../{ => google_drive}/download_from_drive.py | 0 scripts/{ => google_drive}/drive_io.py | 0 4 files changed, 97 insertions(+), 10 deletions(-) rename scripts/{ => google_drive}/download_from_drive.py (100%) rename scripts/{ => google_drive}/drive_io.py (100%) diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index 4eb6579..38670fd 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -26,7 +26,7 @@ load_env() # Load .env file if present (for local dev) -from scripts.drive_io import ( +from scripts.google_drive.drive_io import ( get_drive_service, find_child_folder_id, list_pdfs_in_folder, diff --git a/scripts/full_pipeline.py b/scripts/full_pipeline.py index ed2a94d..95c2de2 100644 --- a/scripts/full_pipeline.py +++ b/scripts/full_pipeline.py @@ -1,12 +1,21 @@ """Full training pipeline: stream ALL PDFs from Drive, extract text, label, train model. -Environment variables: +Modes: + - API mode: Download PDFs from Google Drive and process them + - Local mode: Use PDFs already downloaded locally + +API Mode Environment variables: - GOOGLE_SERVICE_ACCOUNT_JSON (service account JSON string) - GOOGLE_DRIVE_ROOT_FOLDER_ID (root folder containing 'useful' and 'not-useful') - GOOGLE_DRIVE_USE_SHARED_DRIVE=true (if using shared drives / shared folders) +Usage: + - API mode: python full_pipeline.py --api + - Local mode: python full_pipeline.py --local + Behavior: - - Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. + - API mode: Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. + - Local mode: Processes PDFs from specified local directory (expects 'useful' and 'not-useful' subfolders). - Generates labels.json based on folder origin. - Trains model with src/model/train_model.py. """ @@ -15,6 +24,7 @@ import os import json +import argparse from pathlib import Path from typing import Dict import subprocess @@ -29,7 +39,7 @@ load_env() # Load .env file if present (for local dev) -from scripts.drive_io import ( +from scripts.google_drive.drive_io import ( get_drive_service, find_child_folder_id, list_pdfs_in_folder, @@ -52,7 +62,8 @@ def write_labels(labels: Dict[str, str], output_file: Path): json.dump(labels, f, indent=2) -def main(): +def process_api_mode(): + """Download PDFs from Google Drive and process them.""" root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") if not root_id: raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") @@ -68,10 +79,10 @@ def main(): out_dir = Path("data/processed-text") out_dir.mkdir(parents=True, exist_ok=True) labels: Dict[str, str] = {} - + count=1 for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not-useful")]: files = list_pdfs_in_folder(service, folder_id, max_files=None) - print(f"[FULL] Found {len(files)} PDFs in folder label '{label}'") + print(f"Found {len(files)} PDFs in folder label '{label}'") for f in files: pdf_bytes = download_file_bytes(service, f["id"]) text = extract_text_from_pdf_bytes(pdf_bytes) @@ -79,13 +90,89 @@ def main(): txt_name = f"{stem}.txt" (out_dir / txt_name).write_text(text, encoding="utf-8") labels[txt_name] = label + print(f"{count} Processed {f['name']}") + count+=1 write_labels(labels, Path("data/labels.json")) - print(f"[FULL] Wrote {len(labels)} labeled text files. Beginning model training...") + print(f"Wrote {len(labels)} labeled text files.") + + +def process_local_mode(data_path: Path): + """Process PDFs from local directory.""" + if not data_path.exists(): + raise RuntimeError(f"Data path does not exist: {data_path}") + + useful_dir = data_path / "useful" + not_useful_dir = data_path / "not-useful" + + if not useful_dir.exists(): + raise RuntimeError(f"'useful' subfolder not found in {data_path}") + if not not_useful_dir.exists(): + raise RuntimeError(f"'not-useful' subfolder not found in {data_path}") + + out_dir = Path("data/processed-text") + out_dir.mkdir(parents=True, exist_ok=True) + labels: Dict[str, str] = {} + + for folder, label in [(useful_dir, "useful"), (not_useful_dir, "not-useful")]: + pdf_files = list(folder.glob("*.pdf")) + print(f"Found {len(pdf_files)} PDFs in local folder '{label}'") + + for pdf_path in pdf_files: + try: + with open(pdf_path, "rb") as f: + pdf_bytes = f.read() + text = extract_text_from_pdf_bytes(pdf_bytes) + stem = pdf_path.stem + txt_name = f"{stem}.txt" + (out_dir / txt_name).write_text(text, encoding="utf-8") + labels[txt_name] = label + print(f"Processed {pdf_path.name}") + except Exception as e: + print(f"Error processing {pdf_path.name}: {e}") + continue + + write_labels(labels, Path("data/labels.json")) + print(f"Wrote {len(labels)} labeled text files.") - # Train model + +def main(): + parser = argparse.ArgumentParser( + description="Full pipeline: extract text from PDFs and train model", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + API mode: python full_pipeline.py --api + Local mode: python full_pipeline.py --local ./data/pdfs + """ + ) + + # Create mutually exclusive group for --api and --local + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--api", + action="store_true", + help="Use API mode to download PDFs from Google Drive" + ) + group.add_argument( + "--local", + type=Path, + metavar="PATH", + help="Use local mode with PDFs from specified directory (should contain 'useful' and 'not-useful' subfolders)" + ) + + args = parser.parse_args() + + if args.local: + print(f"Running in LOCAL mode with data path: {args.local}") + process_local_mode(args.local) + else: # args.api + print("Running in API mode (Google Drive)") + process_api_mode() + + print("Beginning model training...") run([sys.executable, "src/model/train_model.py"]) - print("[FULL] Training complete.") + print("Training complete.") if __name__ == "__main__": diff --git a/scripts/download_from_drive.py b/scripts/google_drive/download_from_drive.py similarity index 100% rename from scripts/download_from_drive.py rename to scripts/google_drive/download_from_drive.py diff --git a/scripts/drive_io.py b/scripts/google_drive/drive_io.py similarity index 100% rename from scripts/drive_io.py rename to scripts/google_drive/drive_io.py From acd03b6d4bc9f4b28a0545cf065683402a223f96 Mon Sep 17 00:00:00 2001 From: raymondcen Date: Thu, 13 Nov 2025 13:54:17 -0800 Subject: [PATCH 12/15] Revert "feat(model pipeline): modified model pipeline to include api mode" This reverts commit d482a60068be9d5b24b5a264da63b2c5bcb88151. --- scripts/ci_pipeline.py | 2 +- .../{google_drive => }/download_from_drive.py | 0 scripts/{google_drive => }/drive_io.py | 0 scripts/full_pipeline.py | 105 ++---------------- 4 files changed, 10 insertions(+), 97 deletions(-) rename scripts/{google_drive => }/download_from_drive.py (100%) rename scripts/{google_drive => }/drive_io.py (100%) diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index 38670fd..4eb6579 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -26,7 +26,7 @@ load_env() # Load .env file if present (for local dev) -from scripts.google_drive.drive_io import ( +from scripts.drive_io import ( get_drive_service, find_child_folder_id, list_pdfs_in_folder, diff --git a/scripts/google_drive/download_from_drive.py b/scripts/download_from_drive.py similarity index 100% rename from scripts/google_drive/download_from_drive.py rename to scripts/download_from_drive.py diff --git a/scripts/google_drive/drive_io.py b/scripts/drive_io.py similarity index 100% rename from scripts/google_drive/drive_io.py rename to scripts/drive_io.py diff --git a/scripts/full_pipeline.py b/scripts/full_pipeline.py index 95c2de2..ed2a94d 100644 --- a/scripts/full_pipeline.py +++ b/scripts/full_pipeline.py @@ -1,21 +1,12 @@ """Full training pipeline: stream ALL PDFs from Drive, extract text, label, train model. -Modes: - - API mode: Download PDFs from Google Drive and process them - - Local mode: Use PDFs already downloaded locally - -API Mode Environment variables: +Environment variables: - GOOGLE_SERVICE_ACCOUNT_JSON (service account JSON string) - GOOGLE_DRIVE_ROOT_FOLDER_ID (root folder containing 'useful' and 'not-useful') - GOOGLE_DRIVE_USE_SHARED_DRIVE=true (if using shared drives / shared folders) -Usage: - - API mode: python full_pipeline.py --api - - Local mode: python full_pipeline.py --local - Behavior: - - API mode: Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. - - Local mode: Processes PDFs from specified local directory (expects 'useful' and 'not-useful' subfolders). + - Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. - Generates labels.json based on folder origin. - Trains model with src/model/train_model.py. """ @@ -24,7 +15,6 @@ import os import json -import argparse from pathlib import Path from typing import Dict import subprocess @@ -39,7 +29,7 @@ load_env() # Load .env file if present (for local dev) -from scripts.google_drive.drive_io import ( +from scripts.drive_io import ( get_drive_service, find_child_folder_id, list_pdfs_in_folder, @@ -62,8 +52,7 @@ def write_labels(labels: Dict[str, str], output_file: Path): json.dump(labels, f, indent=2) -def process_api_mode(): - """Download PDFs from Google Drive and process them.""" +def main(): root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") if not root_id: raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") @@ -79,10 +68,10 @@ def process_api_mode(): out_dir = Path("data/processed-text") out_dir.mkdir(parents=True, exist_ok=True) labels: Dict[str, str] = {} - count=1 + for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not-useful")]: files = list_pdfs_in_folder(service, folder_id, max_files=None) - print(f"Found {len(files)} PDFs in folder label '{label}'") + print(f"[FULL] Found {len(files)} PDFs in folder label '{label}'") for f in files: pdf_bytes = download_file_bytes(service, f["id"]) text = extract_text_from_pdf_bytes(pdf_bytes) @@ -90,89 +79,13 @@ def process_api_mode(): txt_name = f"{stem}.txt" (out_dir / txt_name).write_text(text, encoding="utf-8") labels[txt_name] = label - print(f"{count} Processed {f['name']}") - count+=1 write_labels(labels, Path("data/labels.json")) - print(f"Wrote {len(labels)} labeled text files.") - - -def process_local_mode(data_path: Path): - """Process PDFs from local directory.""" - if not data_path.exists(): - raise RuntimeError(f"Data path does not exist: {data_path}") - - useful_dir = data_path / "useful" - not_useful_dir = data_path / "not-useful" - - if not useful_dir.exists(): - raise RuntimeError(f"'useful' subfolder not found in {data_path}") - if not not_useful_dir.exists(): - raise RuntimeError(f"'not-useful' subfolder not found in {data_path}") - - out_dir = Path("data/processed-text") - out_dir.mkdir(parents=True, exist_ok=True) - labels: Dict[str, str] = {} - - for folder, label in [(useful_dir, "useful"), (not_useful_dir, "not-useful")]: - pdf_files = list(folder.glob("*.pdf")) - print(f"Found {len(pdf_files)} PDFs in local folder '{label}'") - - for pdf_path in pdf_files: - try: - with open(pdf_path, "rb") as f: - pdf_bytes = f.read() - text = extract_text_from_pdf_bytes(pdf_bytes) - stem = pdf_path.stem - txt_name = f"{stem}.txt" - (out_dir / txt_name).write_text(text, encoding="utf-8") - labels[txt_name] = label - print(f"Processed {pdf_path.name}") - except Exception as e: - print(f"Error processing {pdf_path.name}: {e}") - continue - - write_labels(labels, Path("data/labels.json")) - print(f"Wrote {len(labels)} labeled text files.") + print(f"[FULL] Wrote {len(labels)} labeled text files. Beginning model training...") - -def main(): - parser = argparse.ArgumentParser( - description="Full pipeline: extract text from PDFs and train model", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - API mode: python full_pipeline.py --api - Local mode: python full_pipeline.py --local ./data/pdfs - """ - ) - - # Create mutually exclusive group for --api and --local - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--api", - action="store_true", - help="Use API mode to download PDFs from Google Drive" - ) - group.add_argument( - "--local", - type=Path, - metavar="PATH", - help="Use local mode with PDFs from specified directory (should contain 'useful' and 'not-useful' subfolders)" - ) - - args = parser.parse_args() - - if args.local: - print(f"Running in LOCAL mode with data path: {args.local}") - process_local_mode(args.local) - else: # args.api - print("Running in API mode (Google Drive)") - process_api_mode() - - print("Beginning model training...") + # Train model run([sys.executable, "src/model/train_model.py"]) - print("Training complete.") + print("[FULL] Training complete.") if __name__ == "__main__": From 90037630695197501010391a54a2cba1ad9fd952 Mon Sep 17 00:00:00 2001 From: raymondcen Date: Thu, 13 Nov 2025 13:56:17 -0800 Subject: [PATCH 13/15] feat(model pipeline): modified model pipeline to include api mode --- scripts/ci_pipeline.py | 2 +- scripts/{ => google_drive}/download_from_drive.py | 0 scripts/{ => google_drive}/drive_io.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{ => google_drive}/download_from_drive.py (100%) rename scripts/{ => google_drive}/drive_io.py (100%) diff --git a/scripts/ci_pipeline.py b/scripts/ci_pipeline.py index 4eb6579..38670fd 100644 --- a/scripts/ci_pipeline.py +++ b/scripts/ci_pipeline.py @@ -26,7 +26,7 @@ load_env() # Load .env file if present (for local dev) -from scripts.drive_io import ( +from scripts.google_drive.drive_io import ( get_drive_service, find_child_folder_id, list_pdfs_in_folder, diff --git a/scripts/download_from_drive.py b/scripts/google_drive/download_from_drive.py similarity index 100% rename from scripts/download_from_drive.py rename to scripts/google_drive/download_from_drive.py diff --git a/scripts/drive_io.py b/scripts/google_drive/drive_io.py similarity index 100% rename from scripts/drive_io.py rename to scripts/google_drive/drive_io.py From 3fc2da91619853e8791ea9fb6c437beb420a9d67 Mon Sep 17 00:00:00 2001 From: raymondcen Date: Thu, 13 Nov 2025 13:56:55 -0800 Subject: [PATCH 14/15] feat(model pipeline): included api mode in full pipeline but local mode is much faster --- scripts/full_pipeline.py | 105 +++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 9 deletions(-) diff --git a/scripts/full_pipeline.py b/scripts/full_pipeline.py index ed2a94d..95c2de2 100644 --- a/scripts/full_pipeline.py +++ b/scripts/full_pipeline.py @@ -1,12 +1,21 @@ """Full training pipeline: stream ALL PDFs from Drive, extract text, label, train model. -Environment variables: +Modes: + - API mode: Download PDFs from Google Drive and process them + - Local mode: Use PDFs already downloaded locally + +API Mode Environment variables: - GOOGLE_SERVICE_ACCOUNT_JSON (service account JSON string) - GOOGLE_DRIVE_ROOT_FOLDER_ID (root folder containing 'useful' and 'not-useful') - GOOGLE_DRIVE_USE_SHARED_DRIVE=true (if using shared drives / shared folders) +Usage: + - API mode: python full_pipeline.py --api + - Local mode: python full_pipeline.py --local + Behavior: - - Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. + - API mode: Streams every PDF (no local PDF persistence) and writes extracted text to data/processed-text. + - Local mode: Processes PDFs from specified local directory (expects 'useful' and 'not-useful' subfolders). - Generates labels.json based on folder origin. - Trains model with src/model/train_model.py. """ @@ -15,6 +24,7 @@ import os import json +import argparse from pathlib import Path from typing import Dict import subprocess @@ -29,7 +39,7 @@ load_env() # Load .env file if present (for local dev) -from scripts.drive_io import ( +from scripts.google_drive.drive_io import ( get_drive_service, find_child_folder_id, list_pdfs_in_folder, @@ -52,7 +62,8 @@ def write_labels(labels: Dict[str, str], output_file: Path): json.dump(labels, f, indent=2) -def main(): +def process_api_mode(): + """Download PDFs from Google Drive and process them.""" root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID") if not root_id: raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable") @@ -68,10 +79,10 @@ def main(): out_dir = Path("data/processed-text") out_dir.mkdir(parents=True, exist_ok=True) labels: Dict[str, str] = {} - + count=1 for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not-useful")]: files = list_pdfs_in_folder(service, folder_id, max_files=None) - print(f"[FULL] Found {len(files)} PDFs in folder label '{label}'") + print(f"Found {len(files)} PDFs in folder label '{label}'") for f in files: pdf_bytes = download_file_bytes(service, f["id"]) text = extract_text_from_pdf_bytes(pdf_bytes) @@ -79,13 +90,89 @@ def main(): txt_name = f"{stem}.txt" (out_dir / txt_name).write_text(text, encoding="utf-8") labels[txt_name] = label + print(f"{count} Processed {f['name']}") + count+=1 write_labels(labels, Path("data/labels.json")) - print(f"[FULL] Wrote {len(labels)} labeled text files. Beginning model training...") + print(f"Wrote {len(labels)} labeled text files.") + + +def process_local_mode(data_path: Path): + """Process PDFs from local directory.""" + if not data_path.exists(): + raise RuntimeError(f"Data path does not exist: {data_path}") + + useful_dir = data_path / "useful" + not_useful_dir = data_path / "not-useful" + + if not useful_dir.exists(): + raise RuntimeError(f"'useful' subfolder not found in {data_path}") + if not not_useful_dir.exists(): + raise RuntimeError(f"'not-useful' subfolder not found in {data_path}") + + out_dir = Path("data/processed-text") + out_dir.mkdir(parents=True, exist_ok=True) + labels: Dict[str, str] = {} + + for folder, label in [(useful_dir, "useful"), (not_useful_dir, "not-useful")]: + pdf_files = list(folder.glob("*.pdf")) + print(f"Found {len(pdf_files)} PDFs in local folder '{label}'") + + for pdf_path in pdf_files: + try: + with open(pdf_path, "rb") as f: + pdf_bytes = f.read() + text = extract_text_from_pdf_bytes(pdf_bytes) + stem = pdf_path.stem + txt_name = f"{stem}.txt" + (out_dir / txt_name).write_text(text, encoding="utf-8") + labels[txt_name] = label + print(f"Processed {pdf_path.name}") + except Exception as e: + print(f"Error processing {pdf_path.name}: {e}") + continue + + write_labels(labels, Path("data/labels.json")) + print(f"Wrote {len(labels)} labeled text files.") - # Train model + +def main(): + parser = argparse.ArgumentParser( + description="Full pipeline: extract text from PDFs and train model", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + API mode: python full_pipeline.py --api + Local mode: python full_pipeline.py --local ./data/pdfs + """ + ) + + # Create mutually exclusive group for --api and --local + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--api", + action="store_true", + help="Use API mode to download PDFs from Google Drive" + ) + group.add_argument( + "--local", + type=Path, + metavar="PATH", + help="Use local mode with PDFs from specified directory (should contain 'useful' and 'not-useful' subfolders)" + ) + + args = parser.parse_args() + + if args.local: + print(f"Running in LOCAL mode with data path: {args.local}") + process_local_mode(args.local) + else: # args.api + print("Running in API mode (Google Drive)") + process_api_mode() + + print("Beginning model training...") run([sys.executable, "src/model/train_model.py"]) - print("[FULL] Training complete.") + print("Training complete.") if __name__ == "__main__": From 43ce8b708ec1d0dc2dd129424c50d565e49b7f02 Mon Sep 17 00:00:00 2001 From: raymondcen Date: Thu, 13 Nov 2025 14:08:40 -0800 Subject: [PATCH 15/15] docs: updated contributing.md to match new pipeline --- documentation/CONTRIBUTING.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/documentation/CONTRIBUTING.md b/documentation/CONTRIBUTING.md index 9a15d90..7702319 100644 --- a/documentation/CONTRIBUTING.md +++ b/documentation/CONTRIBUTING.md @@ -27,9 +27,25 @@ All contributors must follow the Oregon State University Student Code of Conduct pip install -r requirements.txt ``` * ### Running the application -``` - python src/main.py -``` + * If you do have the [dataset](https://drive.google.com/drive/u/2/folders/1U3_-TmnXnuBPR9vukkyV-3ITsxPr-nfo) downloaded locally on your machine: + ``` + python scripts/full_pipeline.py --local + ``` + * Note: Your local dataset folder should be formatted like so: + ``` + folder + |-> useful + |-> pdfs + |-> ... + |-> not-useful + |-> pdfs + |-> ... + ``` + * If you do not have the dataset downloaded or cannot have it on your locally on your machine: + ``` + python scripts/full_pipeline.py --api + ``` + * Note: You will need access to the .env file * ### Enviorment Variables * Sensitive information such as API keys will be stored in a local .env file which will be excluded by .gitignore. * Never hardcode secrets