run_processing.py

import os
import sys
import pandas as pd
from pathlib import Path

def process_data():
    # 1. Setup paths
    script_path = Path(__file__).resolve()
    project_root = script_path.parent.parent.parent
    raw_dir = project_root / "data" / "raw"
    processed_dir = project_root / "data" / "processed"

    # 2. Print debug info
    print("\n=== Environment Info ===")
    print(f"Working Directory: {os.getcwd()}")
    print(f"Project Root: {project_root}")
    print(f"Raw Data Dir: {raw_dir}")
    print(f"Processed Data Dir: {processed_dir}")

    # 3. Verify raw data files
    print("\n=== Checking Raw Data Files ===")
    required_files = ['train_transaction.csv', 'train_identity.csv']
    for file in required_files:
        file_path = raw_dir / file
        if file_path.exists():
            size_mb = file_path.stat().st_size / (1024 * 1024)
            print(f"✓ {file} found ({size_mb:.2f} MB)")
        else:
            print(f"✗ {file} missing!")
            return False

    # 4. Create processed directory
    print("\n=== Setting Up Directories ===")
    processed_dir.mkdir(exist_ok=True, parents=True)
    print(f"✓ Created processed directory: {processed_dir}")

    try:
        # 5. Load data
        print("\n=== Loading Data ===")
        train_transaction = pd.read_csv(raw_dir / "train_transaction.csv")
        print(f"✓ Loaded transaction data: {train_transaction.shape}")
        
        train_identity = pd.read_csv(raw_dir / "train_identity.csv")
        print(f"✓ Loaded identity data: {train_identity.shape}")

        # 6. Process data
        print("\n=== Processing Data ===")
        # Calculate split sizes
        total_samples = len(train_transaction)
        train_size = int(0.7 * total_samples)
        val_size = int(0.15 * total_samples)
        
        # Shuffle with fixed seed
        train_transaction = train_transaction.sample(frac=1, random_state=42)
        
        # Match identity data with shuffled transaction data
        train_identity = train_identity.set_index('TransactionID')
        train_identity = train_identity.reindex(train_transaction['TransactionID']).reset_index()
        
        # Create splits
        splits = {
            'train': (0, train_size),
            'val': (train_size, train_size + val_size),
            'test': (train_size + val_size, total_samples)
        }

        # 7. Save splits
        print("\n=== Saving Splits ===")
        for split_name, (start, end) in splits.items():
            # Create split directory
            split_dir = processed_dir / split_name
            split_dir.mkdir(exist_ok=True)
            
            # Get split data
            transaction_split = train_transaction.iloc[start:end]
            identity_split = train_identity.iloc[start:end]
            
            # Save files
            transaction_path = split_dir / 'transaction.csv'
            identity_path = split_dir / 'identity.csv'
            
            transaction_split.to_csv(transaction_path, index=False)
            identity_split.to_csv(identity_path, index=False)
            
            print(f"✓ {split_name}:")
            print(f"  - Transaction: {transaction_split.shape} -> {transaction_path}")
            print(f"  - Identity: {identity_split.shape} -> {identity_path}")

        print("\n=== Processing Complete ===")
        print("✓ All data has been processed and saved successfully!")
        return True

    except Exception as e:
        print(f"\n✗ Error during processing: {str(e)}")
        import traceback
        traceback.print_exc()
        return False

if __name__ == "__main__":
    success = process_data()
    if not success:
        sys.exit(1)