Google drive api

.github/workflows/working_sw.yml

@@ -21,26 +21,27 @@ jobs:
           sudo apt-get install -y tesseract-ocr
           python -m pip install --upgrade pip
           pip install -r requirements.txt
-        # OPTIONAL: Uncomment below after adding GOOGLE_SERVICE_ACCOUNT_JSON secret
-        # (Service Account JSON contents) to repo settings OR loading via .env.
-        # This step would download PDFs from a shared Google Drive folder.
-        # - name: Download training PDFs (Google Drive)
-        #   env:
-        #     GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }}
-        #   run: |
-        #     python scripts/download_from_drive.py
-        #
+
+      # CI pipeline (streams 20 PDFs per class from Drive and trains model)
+      - name: CI pipeline (Drive stream)
+        env:
+          GOOGLE_SERVICE_ACCOUNT_JSON: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_JSON }}
+          GOOGLE_DRIVE_ROOT_FOLDER_ID: ${{ secrets.GOOGLE_DRIVE_ROOT_FOLDER_ID }}
+          PYTHONUNBUFFERED: 1
+        run: |
+          python -u scripts/ci_pipeline.py
+
         # OPTIONAL: Model cache and artifacts (commented until enabled)
-        # - name: Cache model artifacts
-        #   uses: actions/cache@v3
-        #   with:
-        #     path: |
-        #       src/model/models
-        #       ~/.joblib
-        #       ~/.sklearn
-        #     key: ${{ runner.os }}-sklearn-model-${{ hashFiles('src/model/**/*.py') }}
-        #     restore-keys: |
-        #       ${{ runner.os }}-sklearn-model-
+      - name: Cache model artifacts
+        uses: actions/cache@v3
+        with:
+          path: |
+            src/model/models
+            ~/.joblib
+            ~/.sklearn
+          key: ${{ runner.os }}-sklearn-model-${{ hashFiles('src/model/**/*.py') }}
+          restore-keys: |
+            ${{ runner.os }}-sklearn-model-
 
       # Code quality checks
       - name: Check code formatting (Black)
@@ -55,24 +56,22 @@ jobs:
           echo "Flake8 check completed."
 
       # Data preprocessing pipeline
-      - name: Run PDF extraction script
+      # Optional: still run a single-file extraction test on repo test asset
+      - name: Run PDF extraction script (repo test asset)
         run: |
-          mkdir -p data/processed-text
-          mkdir -p data/useful
-          mkdir -p data/not-useful
-          mkdir -p data/needs-check
           INPUT_PDF="tests/test.pdf"
           python src/preprocessing/pdf_text_extraction.py "$INPUT_PDF"
 
-      - name: Generate labels
-        run: |
-          echo "Generating labels for extracted text..."
-          python src/preprocessing/generate_labels.py
+      # - name: Generate labels
+      #   run: |
+      #     echo "Generating labels for extracted text..."
+      #     python src/preprocessing/generate_labels.py
 
-      - name: Load and preprocess data
-        run: |
-          echo "Loading and preprocessing dataset..."
-          python src/preprocessing/data_loader.py
+      # - name: Load and preprocess data
+      #   run: |
+      #     echo "Loading and preprocessing dataset..."
+      #     python src/preprocessing/data_loader.py
+      # No full pipeline here; use full_training_pipeline.py on main or a scheduled workflow
 
       - name: Validate preprocessing pipeline
         run: |
@@ -92,12 +91,7 @@ jobs:
           python -c "import json; f=open('data/labels.json'); json.load(f); f.close()" || exit 1
 
       # Model workflow temporarily disabled (training & classification skipped)
-      # - name: Train model
-      #   run: |
-      #     python src/model/train_model.py
-      # - name: Test model classification
-      #   run: |
-      #     python src/model/pdf_classifier.py --pdf-path "tests/test.pdf" --model_dir "src/model/models"
+      # Skip model training in CI for speed; exercised in full-training job on main/schedule
 
       # Unit testing and coverage
       - name: Run unit tests with coverage
@@ -114,13 +108,13 @@ jobs:
           path: htmlcov/
 
       # OPTIONAL: Upload trained model artifacts (enable once model steps are active)
-      # - name: Upload trained model
-      #   uses: actions/upload-artifact@v4
-      #   with:
-      #     name: trained-model
-      #     path: |
-      #       src/model/models/pdf_classifier_model.pkl
-      #       src/model/models/tfidf_vectorizer.pkl
+      - name: Upload trained model
+        uses: actions/upload-artifact@v4
+        with:
+          name: trained-model
+          path: |
+            src/model/models/pdf_classifier_model.pkl
+            src/model/models/tfidf_vectorizer.pkl
 
       - name: Show pipeline summary
         run: |
@@ -129,4 +123,5 @@ jobs:
           ls -l data/processed-text/
           echo "2. Label Generation Results:"
           ls -l data/labels.json
-          echo "(Model steps skipped in this run)"
+          echo "3. Model Training Results:"
+          ls -l src/model/models/

documentation/CONTRIBUTING.md

@@ -27,9 +27,25 @@ All contributors must follow the Oregon State University Student Code of Conduct
     pip install -r requirements.txt
 ```
 * ### Running the application
-```
-    python src/main.py
-```
+  * If you do have the [dataset](https://drive.google.com/drive/u/2/folders/1U3_-TmnXnuBPR9vukkyV-3ITsxPr-nfo) downloaded locally on your machine:
+    ```
+    python scripts/full_pipeline.py --local <file_path_to_dataset>
+    ```
+    * Note: Your local dataset folder should be formatted like so:
+      ```
+      folder
+        |-> useful
+            |-> pdfs
+            |-> ...
+        |-> not-useful
+            |-> pdfs
+            |-> ...
+      ```
+  * If you do not have the dataset downloaded or cannot have it on your locally on your machine:
+    ```
+    python scripts/full_pipeline.py --api
+    ```
+    * Note: You will need access to the .env file
 * ### Enviorment Variables
     * Sensitive information such as API keys will be stored in a local .env file which will be excluded by .gitignore.
     * Never hardcode secrets

requirements.txt

@@ -8,4 +8,8 @@ flake8==6.1.0
 black==25.9.0
 scikit-learn
 dotenv
-numpy
+google-auth 
+google-auth-oauthlib 
+google-auth-httplib2
+numpy
+google-api-python-client

scripts/ci_pipeline.py

@@ -0,0 +1,102 @@
+"""CI pipeline: stream a small, fixed sample from Google Drive and run extraction.
+
+Requirements:
+ - Env GOOGLE_SERVICE_ACCOUNT_JSON set to Service Account JSON string.
+ - Env GOOGLE_DRIVE_ROOT_FOLDER_ID set to the Drive folder containing 'useful' and 'not-useful'.
+ - Optional env CI_FILES_PER_CLASS (default 1).
+
+This script DOES NOT save PDFs locally. It streams bytes and writes extracted text
+into data/processed-text/*.txt, and writes data/labels.json. No training.
+"""
+
+from __future__ import annotations
+
+import os
+import json
+from pathlib import Path
+from typing import Dict, List
+import subprocess
+
+import sys
+from pathlib import Path as _Path
+
+sys.path.append(str(_Path(__file__).resolve().parents[1]))  # add repo root to sys.path
+
+from scripts.env_loader import load_env
+
+load_env()  # Load .env file if present (for local dev)
+
+from scripts.google_drive.drive_io import (
+    get_drive_service,
+    find_child_folder_id,
+    list_pdfs_in_folder,
+    download_file_bytes,
+    sanitize_filename,
+)
+from src.preprocessing.pdf_text_extraction import extract_text_from_pdf_bytes
+
+
+def write_labels(labels: Dict[str, str], output_file: Path):
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+    with output_file.open("w", encoding="utf-8") as f:
+        json.dump(labels, f, indent=2)
+
+
+def main():
+    root_id = os.environ.get("GOOGLE_DRIVE_ROOT_FOLDER_ID")
+    if not root_id:
+        raise RuntimeError("Missing GOOGLE_DRIVE_ROOT_FOLDER_ID environment variable")
+
+    per_class = 5  # Fixed set size for CI
+
+    print("Connecting to Google Drive...")
+    service = get_drive_service()
+    print("Connected to Google Drive")
+
+    print(f"Looking for 'useful' and 'not-useful' folders under root {root_id}...")
+    useful_id = find_child_folder_id(service, root_id, "useful")
+    not_useful_id = find_child_folder_id(service, root_id, "not-useful")
+    if not useful_id:
+        raise RuntimeError(f"Could not find 'useful' subfolder under root folder {root_id}")
+    if not not_useful_id:
+        raise RuntimeError(f"Could not find 'not-useful' subfolder under root folder {root_id}")
+    print(f"Found both folders (useful: {useful_id}, not-useful: {not_useful_id})")
+
+    # Prepare output directory
+    out_dir = Path("data/processed-text")
+    out_dir.mkdir(parents=True, exist_ok=True)
+    print(f"Output directory ready: {out_dir}")
+
+    labels: Dict[str, str] = {}
+
+    for folder_id, label in [(useful_id, "useful"), (not_useful_id, "not useful")]:
+        print(f"\nProcessing '{label}' folder (max {per_class} PDFs)...")
+        files = list_pdfs_in_folder(service, folder_id, max_files=per_class)
+        print(f"Found {len(files)} PDFs in '{label}' folder")
+        for idx, f in enumerate(files, 1):
+            pdf_name = f.get("name", f.get("id", "file"))
+            print(f"[{idx}/{len(files)}] Processing: {pdf_name}")
+            pdf_bytes = download_file_bytes(service, f["id"])
+            print(f"Downloaded {len(pdf_bytes)} bytes")
+            text = extract_text_from_pdf_bytes(pdf_bytes)
+            stem = sanitize_filename(pdf_name)
+            txt_name = f"{stem}.txt"
+            (out_dir / txt_name).write_text(text, encoding="utf-8")
+            labels[txt_name] = label
+            print(f"Extracted {len(text)} chars to {txt_name}")
+
+    write_labels(labels, Path("data/labels.json"))
+    print(f"\nWrote {len(labels)} labels to data/labels.json")
+    print(f"Extracted {len(labels)} text files to {out_dir}")
+
+    # Train model on the CI sample
+    print("\nStarting model training on CI sample...")
+    r = subprocess.run([sys.executable, "src/model/train_model.py"], env={**os.environ, "CI_TRAIN": "1"})
+    if r.returncode != 0:
+        print("Model training failed")
+        raise SystemExit(r.returncode)
+    print("Model training complete")
+
+
+if __name__ == "__main__":
+    main()

scripts/env_loader.py

@@ -0,0 +1,21 @@
+"""Load environment variables from .env file for local development."""
+
+from pathlib import Path
+
+
+def load_env():
+    """Load .env file from repo root if it exists."""
+    env_file = Path(__file__).resolve().parents[1] / ".env"
+    if not env_file.exists():
+        return  # No .env file, assume env vars set manually or in CI
+
+    import os
+
+    with env_file.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            if "=" in line:
+                key, value = line.split("=", 1)
+                os.environ.setdefault(key.strip(), value.strip())