LEDFileMover.py

#!/usr/bin/env python3
"""
LED File Mover - Production File Migration Tool
TRUE MOVE operations with version history preservation

⚠️ CURRENTLY CONFIGURED FOR QA TESTING ONLY ⚠️
- Source: /sites/LEDDocuments (TEST SITE with ERROR_TEST_SAMPLE.pdf)
- Production /sites/Documents is OFF-LIMITS during QA phase

Based on LEDFileCopier_Enhanced.py but uses Microsoft Graph MOVE API
- Preserves version history and metadata
- Atomic move operations (safer than copy+delete)
- Same archive detection and granular processing
- Files disappear from source and appear in destination
"""

from dotenv import load_dotenv
import os
import requests
import json
import logging
import time
from datetime import datetime, timedelta
from msal import ConfidentialClientApplication
import pandas as pd
from urllib.parse import quote
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
import queue

load_dotenv()

# Environment variables
TENANT_ID = os.getenv("TENANT_ID")
CLIENT_ID = os.getenv("CLIENT_ID")
CLIENT_SECRET = os.getenv("CLIENT_SECRET")

# SharePoint sites - USING TEST SITE FOR QA TESTING
SOURCE_SITE = "ledconnection2015.sharepoint.com"
SOURCE_SITE_PATH = "/sites/LEDDocuments"  # TEST SITE - Contains ERROR_TEST_SAMPLE.pdf
SOURCE_FOLDER = ""  # Root of the drive for LEDDocuments
DEST_SITE = "ledconnection2015.sharepoint.com"
DEST_SITE_PATH = "/sites/DhilenTestProjectPage"

class LEDFileMover:
    def __init__(self, mapping_file_path, daily_folders_to_process=None):
        """Initialize LED File Mover with scheduled daily migration"""
        self.mapping_file_path = mapping_file_path
        self.folder_mapping = {}
        self.access_token = None
        
        # Scheduled daily folder processing - REPLACE THIS LIST FOR EACH MIGRATION DAY
        self.daily_folders_to_process = daily_folders_to_process or [
            # Example folder list - replace with your daily migration folders
            # "ConnectWIse Today",
            # "Sales",
            # "Marketing"
        ]
        
        # Session tracking for migration history
        self.session_id = f"migration_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
        self.session_start_time = datetime.now()
        
        # Pre-archived folders (automatically marked as processed)
        self.archived_folders_preloaded = {
            "2020 Preferred Vendors", "2021 Spec Sheet Revamp", "2022 Operations", 
            "Constant Improvement", "Design app -Dino", "Distribution Pricing", 
            "Domestic Stock", "Edited Spec Sheets", "Extended Warranty Program", 
            "First Class Vending", "Form Uploads", "Growthwise Partners", 
            "LC-Engineering", "LED Connection Employee Contact_Extension List", 
            "LED Connection Spec Sheets (New Format)", "Lightfair 2022", 
            "Lighting Audit Services (B. Bridges Share)", "Luna Lighting Shared Folder", 
            "MicroSourcing Share", "Monday.com", "NV Energy", "Neon Production", 
            "OEM", "OEM IES Files", "Optimus next gen", "Our Spec Sheets", 
            "Personal Folders", "Photometrics & Lighting Design", 
            "Power Bi view - for the app", "Projected Invoice Report", "Projects", 
            "Purchasing", "SHARE - GRAINGER & KSC", "SHARE WITH LITERITE", 
            "SHARED - Kroger - Anthony Fredenberg", "Skagit Valley Square - PSE P1132221.1", 
            "Spec Sheets for Technical Review", "Subcontractor Management", 
            "Supply Chain", "Turnkey", "VSB Consulting SHARE", "YESCO Shared Folder", 
            "z - General Archive", "zzzTristanKroger"
        }
        
        # Session statistics
        self.session_stats = {
            "folders_completed_this_session": 0,
            "files_moved_this_session": 0,
            "files_archived_this_session": 0,
            "errors_this_session": 0
        }
        self.token_expires_at = 0
        
        # SharePoint drive IDs (will be set during initialization)
        self.source_drive_id = None
        self.dest_drive_id = None
        
        # Archive detection - FIXED DATE to match Production Documents Plan
        # Using exact same threshold as PRODUCTION_Documents_Plan_20250730_143811.json
        self.archive_threshold_date = datetime.fromisoformat("2023-07-30T13:54:18")
        self.archive_threshold_days = (datetime.now() - self.archive_threshold_date).days  # For display purposes
        
        # Performance settings
        self.max_workers = 3  # Conservative for MOVE operations
        self.move_delay = 0.2  # Slightly longer delay for MOVE operations
        self.batch_save_frequency = 25  # Save more frequently for moves
        self.request_timeout = 45  # Longer timeout for MOVE operations
        
        # Progress tracking
        self.start_time = time.time()
        self.files_moved_this_session = 0
        self.folders_archived_this_session = 0
        
        # Persistent storage files
        self.moved_files_db = "moved_files_database.json"
        self.failed_moves_report = "failed_moves_report.json"
        self.archived_folders_log = "archived_folders_registry.json"
        self.move_mapping_log = "move_audit_trail.json"
        
        # Thread safety locks for shared data structures
        self.moved_files_lock = threading.Lock()
        self.failed_moves_lock = threading.Lock()
        self.archived_folders_lock = threading.Lock()
        self.move_mappings_lock = threading.Lock()
        self.session_stats_lock = threading.Lock()
        
        # Load existing progress
        self.moved_files = self.load_moved_files()
        self.failed_moves = self.load_failed_moves()
        self.archived_folders = self.load_archived_folders()
        self.move_mappings = self.load_move_mappings()
        
        # Setup logging
        self.setup_logging()
        
        # Load folder mapping
        self.load_folder_mapping()
        
        self.logger.info(f"Loaded {len(self.folder_mapping)} folder mappings for MOVE operation")
    
    def setup_logging(self):
        """Setup logging for file mover"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('led_file_mover.log'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)
    
    def load_folder_mapping(self):
        """Load folder mapping from Excel file"""
        try:
            df = pd.read_excel(self.mapping_file_path)
            for _, row in df.iterrows():
                source_folder = row['Current Folder (LED Main - 2019)'].strip()
                dest_folder = row['Destination'].strip()
                self.folder_mapping[source_folder] = dest_folder
        except Exception as e:
            self.logger.error(f"Error loading folder mapping: {e}")
            raise
    
    def get_access_token(self, force_refresh=False):
        """Get Microsoft Graph API access token"""
        current_time = time.time()
        
        if not force_refresh and self.access_token and current_time < self.token_expires_at - 300:
            return self.access_token
        
        try:
            authority = f"https://login.microsoftonline.com/{TENANT_ID}"
            app = ConfidentialClientApplication(
                CLIENT_ID,
                authority=authority,
                client_credential=CLIENT_SECRET
            )
            
            scopes = ["https://graph.microsoft.com/.default"]
            result = app.acquire_token_for_client(scopes=scopes)
            
            if "access_token" in result:
                self.access_token = result["access_token"]
                self.token_expires_at = current_time + result.get("expires_in", 3600)
                self.logger.info("Access token refreshed")
                return self.access_token
            else:
                self.logger.error(f"Error getting token: {result}")
                return None
                
        except Exception as e:
            self.logger.error(f"Exception getting access token: {e}")
            return None
    
    def initialize_drives(self):
        """Initialize SharePoint drive IDs"""
        try:
            if not self.get_access_token():
                return False
            
            # Get source drive ID
            source_url = f"https://graph.microsoft.com/v1.0/sites/{SOURCE_SITE}:{SOURCE_SITE_PATH}:/drive"
            response = self.make_graph_request(source_url)
            if response and response.status_code == 200:
                self.source_drive_id = response.json()["id"]
            else:
                self.logger.error("Failed to get source drive ID")
                return False
            
            # Get destination drive ID
            dest_url = f"https://graph.microsoft.com/v1.0/sites/{DEST_SITE}:{DEST_SITE_PATH}:/drive"
            response = self.make_graph_request(dest_url)
            if response and response.status_code == 200:
                self.dest_drive_id = response.json()["id"]
            else:
                self.logger.error("Failed to get destination drive ID")
                return False
            
            self.logger.info(f"Initialized drives - Source: {self.source_drive_id}, Dest: {self.dest_drive_id}")
            return True
            
        except Exception as e:
            self.logger.error(f"Error initializing drives: {e}")
            return False
    
    def make_graph_request(self, url, method="GET", data=None, headers=None, timeout=None):
        """Make Microsoft Graph API request with error handling"""
        if not timeout:
            timeout = self.request_timeout
            
        max_retries = 3
        
        for attempt in range(max_retries):
            try:
                if not headers:
                    headers = {
                        'Authorization': f'Bearer {self.access_token}',
                        'Content-Type': 'application/json'
                    }
                
                if method == "GET":
                    response = requests.get(url, headers=headers, timeout=timeout)
                elif method == "POST":
                    response = requests.post(url, headers=headers, json=data, timeout=timeout)
                elif method == "PUT":
                    response = requests.put(url, headers=headers, data=data, timeout=timeout)
                elif method == "PATCH":
                    response = requests.patch(url, headers=headers, json=data, timeout=timeout)
                else:
                    response = requests.request(method, url, headers=headers, json=data, timeout=timeout)
                
                if response.status_code == 429:  # Rate limited
                    retry_after = int(response.headers.get('Retry-After', 1))
                    self.logger.warning(f"Rate limited, waiting {retry_after} seconds")
                    time.sleep(retry_after)
                    continue
                elif response.status_code == 401:  # Token expired
                    self.logger.warning("Token expired, refreshing...")
                    if self.get_access_token(force_refresh=True):
                        headers['Authorization'] = f'Bearer {self.access_token}'
                        continue
                    else:
                        break
                else:
                    return response
                    
            except requests.exceptions.Timeout:
                self.logger.warning(f"Request timeout, retrying... ({attempt+1}/{max_retries})")
                if attempt == max_retries - 1:
                    return None
            except Exception as e:
                self.logger.error(f"Request error: {e}")
                if attempt == max_retries - 1:
                    return None
        
        return None
    
    def move_file_with_version_history(self, source_file_info, dest_folder_path, source_folder_name):
        """Move file using Microsoft Graph MOVE API - preserves version history"""
        try:
            # Get destination folder ID
            dest_folder_id = self.get_folder_id_by_path(dest_folder_path)
            if not dest_folder_id:
                self.log_move_mapping(
                    source_path=source_file_info['path'],
                    dest_path=f"{dest_folder_path}/{source_file_info['name']}",
                    action="move_failed",
                    status="error",
                    additional_info={"error": "Could not get destination folder ID"}
                )
                return False
            
            # MOVE operation using Microsoft Graph PATCH API
            move_url = f"https://graph.microsoft.com/v1.0/drives/{self.source_drive_id}/items/{source_file_info['id']}"
            
            move_data = {
                "parentReference": {
                    "driveId": self.dest_drive_id,
                    "id": dest_folder_id
                }
                # Note: Don't change name unless necessary - preserves filename
            }
            
            response = self.make_graph_request(move_url, method="PATCH", data=move_data)
            
            if response and response.status_code == 200:
                moved_file_info = response.json()
                
                # SUCCESS - Log successful file move
                self.log_move_mapping(
                    source_path=source_file_info['path'],
                    dest_path=f"{dest_folder_path}/{source_file_info['name']}",
                    action="moved_with_history",
                    status="success",
                    additional_info={
                        "source_folder": source_folder_name,
                        "file_size_bytes": source_file_info.get('size', 0),
                        "last_modified": source_file_info.get('last_modified'),
                        "move_timestamp": datetime.now().isoformat(),
                        "version_history_preserved": True,
                        "new_item_id": moved_file_info.get('id')
                    }
                )
                
                self.logger.info(f"✅ MOVED (with history): {source_file_info['name']}")
                return True
                
            else:
                error_msg = f"Move failed: {response.status_code if response else 'No response'}"
                self.log_move_mapping(
                    source_path=source_file_info['path'],
                    dest_path=f"{dest_folder_path}/{source_file_info['name']}",
                    action="move_failed",
                    status="error",
                    additional_info={
                        "error": error_msg,
                        "response_code": response.status_code if response else None
                    }
                )
                
                self.logger.warning(f"❌ MOVE FAILED: {source_file_info['name']} - {error_msg}")
                return False
                
        except Exception as e:
            self.log_move_mapping(
                source_path=source_file_info['path'],
                dest_path=f"{dest_folder_path}/{source_file_info.get('name', 'unknown')}",
                action="move_failed",
                status="error",
                additional_info={
                    "error": str(e),
                    "stage": "exception"
                }
            )
            
            self.logger.error(f"❌ MOVE EXCEPTION: {source_file_info.get('name', 'unknown')} - {str(e)}")
            return False
    
    def get_folder_id_by_path(self, folder_path):
        """Get SharePoint folder ID by path, create if it doesn't exist"""
        try:
            if not folder_path or folder_path == "":
                # Root folder
                return "root"
            
            # Try to get existing folder
            encoded_path = quote(folder_path)
            url = f"https://graph.microsoft.com/v1.0/drives/{self.dest_drive_id}/root:/{encoded_path}"
            
            response = self.make_graph_request(url)
            
            if response and response.status_code == 200:
                return response.json()["id"]
            elif response and response.status_code == 404:
                # Folder doesn't exist, create it
                return self.create_folder_path_recursive(folder_path)
            else:
                self.logger.error(f"Error getting folder ID for {folder_path}: {response.status_code if response else 'No response'}")
                return None
                
        except Exception as e:
            self.logger.error(f"Exception getting folder ID for {folder_path}: {e}")
            return None
    
    def create_folder_path_recursive(self, folder_path):
        """Recursively create folder path and return final folder ID"""
        try:
            parts = folder_path.split('/')
            current_path = ""
            current_id = "root"
            
            for part in parts:
                if not part:  # Skip empty parts
                    continue
                
                current_path = f"{current_path}/{part}" if current_path else part
                
                # Check if this level exists
                encoded_path = quote(current_path)
                check_url = f"https://graph.microsoft.com/v1.0/drives/{self.dest_drive_id}/root:/{encoded_path}"
                
                response = self.make_graph_request(check_url)
                
                if response and response.status_code == 200:
                    current_id = response.json()["id"]
                elif response and response.status_code == 404:
                    # Create this folder
                    create_url = f"https://graph.microsoft.com/v1.0/drives/{self.dest_drive_id}/items/{current_id}/children"
                    
                    folder_data = {
                        "name": part,
                        "folder": {}
                    }
                    
                    create_response = self.make_graph_request(create_url, method="POST", data=folder_data)
                    
                    if create_response and create_response.status_code == 201:
                        current_id = create_response.json()["id"]
                        self.logger.info(f"✅ Created folder: {current_path}")
                    else:
                        self.logger.error(f"Failed to create folder {current_path}: {create_response.status_code if create_response else 'No response'}")
                        return None
                else:
                    self.logger.error(f"Error checking folder {current_path}: {response.status_code if response else 'No response'}")
                    return None
            
            return current_id
            
        except Exception as e:
            self.logger.error(f"Exception creating folder path {folder_path}: {e}")
            return None
    

    def get_daily_folders_to_process(self):
        """Get list of folders scheduled for today's migration"""
        scheduled_folders = []
        
        for folder_name in self.daily_folders_to_process:
            if folder_name in self.folder_mapping:
                dest_folder = self.folder_mapping[folder_name]
                scheduled_folders.append((folder_name, dest_folder))
            else:
                self.logger.warning(f"⚠️ Scheduled folder '{folder_name}' not found in mapping file")
        
        return scheduled_folders
    
    def log_session_start(self):
        """Log session start information"""
        session_log = {
            "session_id": self.session_id,
            "session_start": self.session_start_time.isoformat(),
            "daily_folders_scheduled": self.daily_folders_to_process,
            "total_folders_scheduled": len(self.daily_folders_to_process),
            "pre_archived_folders_count": len(self.archived_folders_preloaded),
            "archive_threshold_date": self.archive_threshold_date.isoformat()
        }
        
        session_log_file = f"migration_session_{self.session_id}.json"
        with open(session_log_file, 'w') as f:
            json.dump(session_log, f, indent=2)
        
        self.logger.info(f"📝 Session log created: {session_log_file}")
        return session_log_file
    
    def log_session_end(self):
        """Log session completion information"""
        session_end_time = datetime.now()
        duration = session_end_time - self.session_start_time
        
        session_summary = {
            "session_id": self.session_id,
            "session_start": self.session_start_time.isoformat(),
            "session_end": session_end_time.isoformat(),
            "session_duration_hours": duration.total_seconds() / 3600,
            "daily_folders_scheduled": self.daily_folders_to_process,
            "session_stats": self.session_stats
        }
        
        session_log_file = f"migration_session_{self.session_id}.json"
        with open(session_log_file, 'w') as f:
            json.dump(session_summary, f, indent=2)
        
        self.logger.info(f"📝 Session completed and logged: {session_log_file}")
        return session_summary

    def move_all_mapped_folders_with_archive_detection(self):
        """Move scheduled daily folders with enhanced multi-threading - SCHEDULED MODE"""
        self.logger.info(f"🚀 Starting SCHEDULED DAILY MIGRATION with enhanced threading")
        self.logger.info(f"📁 Total folders in mapping: {len(self.folder_mapping)}")
        self.logger.info(f"📅 Daily folders scheduled: {len(self.daily_folders_to_process)}")
        self.logger.info(f"🗃️ Pre-archived folders: {len(self.archived_folders_preloaded)}")
        self.logger.info(f"🗃️ Archive threshold: Files older than {self.archive_threshold_date.strftime('%Y-%m-%d')}")
        
        # Log session start
        self.log_session_start()
        
        print(f"\n📅 SCHEDULED DAILY MIGRATION MODE")
        print(f"   Processing {len(self.daily_folders_to_process)} scheduled folders in order")
        print(f"   Session ID: {self.session_id}")
        print("=" * 70)
        
        # Validate scheduled folders exist in mapping
        scheduled_folders = self.get_daily_folders_to_process()
        
        if not scheduled_folders:
            self.logger.warning("⚠️ No valid folders found in daily schedule")
            print("❌ No folders scheduled or found in mapping file")
            self.log_session_end()
            return 0, 0
        
        print(f"📂 SCHEDULED FOLDERS:")
        for i, (source, dest) in enumerate(scheduled_folders, 1):
            print(f"  {i}. {source} → {dest}")
        print("=" * 70)
        
        total_success = 0
        total_errors = 0
        
        # Process each scheduled folder in order
        for folder_index, (source_folder, dest_folder) in enumerate(scheduled_folders, 1):
            try:
                print(f"\n🔄 PROCESSING FOLDER {folder_index}/{len(scheduled_folders)}")
                print(f"📁 Source: {source_folder}")
                print(f"📁 Destination: {dest_folder}")
                print(f"🧵 Using {self.max_workers} workers focusing on this folder")
                
                # Process folder with enhanced threading
                success, errors = self.move_folder_with_archive_detection(source_folder, dest_folder)
                
                # Update session statistics
                self.session_stats["files_moved_this_session"] += success
                self.session_stats["errors_this_session"] += errors
                self.session_stats["folders_completed_this_session"] += 1
                
                total_success += success
                total_errors += errors
                
                print(f"✅ COMPLETED {source_folder}: {success} files moved, {errors} errors")
                print(f"📊 Session Progress: {folder_index}/{len(scheduled_folders)} folders completed")
                
                # Save progress after each folder
                self.save_all_progress()
                
            except Exception as e:
                self.logger.error(f"❌ Error processing scheduled folder {source_folder}: {e}")
                self.session_stats["errors_this_session"] += 1
                total_errors += 1
                print(f"❌ Failed to process {source_folder}: {e}")
        
        print(f"\n🎉 SCHEDULED MIGRATION COMPLETED")
        print(f"📊 Session Summary:")
        print(f"   - Folders processed: {self.session_stats['folders_completed_this_session']}")
        print(f"   - Files moved: {self.session_stats['files_moved_this_session']}")
        print(f"   - Errors: {self.session_stats['errors_this_session']}")
        print(f"   - Session ID: {self.session_id}")
        
        # Log session completion
        self.log_session_end()
        
        # Final save and statistics
        self.save_all_progress()
        elapsed_time = time.time() - self.start_time
        
        self.logger.info(f"🏁 MOVE OPERATION COMPLETED")
        self.logger.info(f"📊 Results: {total_success} files moved, {total_errors} errors")
        self.logger.info(f"🗃️ Archived: {self.session_stats['files_archived_this_session']} folders")
        self.logger.info(f"⏱️ Time: {elapsed_time/3600:.1f} hours")
        
        return total_success, total_errors
    
    def move_folder_with_archive_detection(self, source_folder, dest_folder):
        """Move folder with GRANULAR archive detection - MOVE VERSION"""
        self.logger.info(f"🔍 Starting MOVE with granular archive detection: {source_folder} -> {dest_folder}")
        
        # Step 1: Check if entire top-level folder should be archived
        source_path = source_folder if not SOURCE_FOLDER else f"{SOURCE_FOLDER}/{source_folder}"
        is_archived, archive_reason = self.is_folder_archived(source_path)
        
        if is_archived:
            # Log archived folder
            archived_entry = {
                "folder_name": source_folder,
                "source_path": source_path,
                "archive_reason": archive_reason,
                "archive_threshold_date": self.archive_threshold_date.isoformat(),
                "timestamp": datetime.now().isoformat()
            }
            
            with self.archived_folders_lock:
                if "archived_folders" not in self.archived_folders:
                    self.archived_folders["archived_folders"] = []
                
                self.archived_folders["archived_folders"].append(archived_entry)
                
                # Update summary  
                self.archived_folders["summary"] = {
                    "total_archived": len(self.archived_folders["archived_folders"]),
                    "last_updated": datetime.now().isoformat(),
                    "archive_threshold_date": self.archive_threshold_date.isoformat(),
                    "folders_archived_this_session": len(self.archived_folders.get("archived_folders", []))
                }
            
            # Update session statistics
            self.session_stats["files_archived_this_session"] += 1
            
            # Log move mapping for archived folder
            self.log_move_mapping(
                source_path=source_path,
                dest_path="ARCHIVED (not moved)",
                action="archived",
                status="skipped",
                additional_info={
                    "reason": archive_reason,
                    "threshold_date": self.archive_threshold_date.isoformat()
                }
            )
            
            self.logger.info(f"📁 ARCHIVED: {source_folder} - {archive_reason}")
            return 0, 0  # No files moved
        
        # Step 2: Top-level folder is not archived, use granular subfolder processing
        self.logger.info(f"🔄 Processing ACTIVE folder with granular archive detection: {source_folder} -> {dest_folder}")
        return self.move_folder_with_granular_archive_detection(source_path, dest_folder, source_folder)
    
    def move_folder_with_granular_archive_detection(self, source_path, dest_folder, source_folder_name):
        """Move folder with granular archive detection at every nested level - MOVE VERSION"""
        self.logger.info(f"🔍 Starting granular MOVE processing: {source_path}")
        
        # Use recursive processing similar to Enhanced version but with MOVE operations
        return self.process_folder_recursively_with_archive_detection_move(
            current_folder_path=source_path,
            dest_base_folder=dest_folder,
            source_folder_name=source_folder_name,
            relative_path=""
        )
    
    def process_folder_recursively_with_archive_detection_move(self, current_folder_path, dest_base_folder, source_folder_name, relative_path):
        """Recursively process folder with archive detection for MOVE operations"""
        try:
            # STEP 1: Check if THIS folder (including all subfolders) should be archived
            all_files_in_folder = self.get_files_in_folder_with_dates(current_folder_path)
            
            if all_files_in_folder:
                is_this_folder_archived = self.check_if_all_files_archived(all_files_in_folder)
                
                if is_this_folder_archived:
                    # This entire folder is archived - skip moving it
                    folder_display_name = f"{source_folder_name}/{relative_path}" if relative_path else source_folder_name
                    
                    oldest_file = min(all_files_in_folder, key=lambda x: x.get('last_modified', ''))
                    youngest_file = max(all_files_in_folder, key=lambda x: x.get('last_modified', ''))
                    
                    archived_entry = {
                        "folder_name": folder_display_name,
                        "source_path": current_folder_path,
                        "archive_reason": f"Subfolder archived: All {len(all_files_in_folder)} files older than 2 years. Oldest: {oldest_file['last_modified']}, Youngest: {youngest_file['last_modified']}",
                        "archive_threshold_date": self.archive_threshold_date.isoformat(),
                        "timestamp": datetime.now().isoformat(),
                        "type": "granular_subfolder_archive"
                    }
                    
                    with self.archived_folders_lock:
                        if "archived_folders" not in self.archived_folders:
                            self.archived_folders["archived_folders"] = []
                        
                        self.archived_folders["archived_folders"].append(archived_entry)
                    
                    # Update session statistics
                    self.session_stats["files_archived_this_session"] += 1
                    
                    self.logger.info(f"📁 ARCHIVED SUBFOLDER: {folder_display_name} - All {len(all_files_in_folder)} files older than 2 years")
                    return 0, 0  # No files moved from archived folder
            
            # STEP 2: This folder is NOT archived, process its contents
            total_success = 0
            total_errors = 0
            
            # Get immediate files and subfolders
            immediate_files, immediate_subfolders = self.get_immediate_folder_contents(current_folder_path)
            
            # STEP 3: Move immediate files in this folder
            if immediate_files:
                dest_path = f"{dest_base_folder}/{relative_path}" if relative_path else dest_base_folder
                success, errors = self.move_subfolder_files(immediate_files, dest_path, f"{source_folder_name}/{relative_path}" if relative_path else source_folder_name)
                total_success += success
                total_errors += errors
            
            # STEP 4: Recursively process each immediate subfolder
            for subfolder_name in immediate_subfolders:
                new_relative_path = f"{relative_path}/{subfolder_name}" if relative_path else subfolder_name
                new_current_path = f"{current_folder_path}/{subfolder_name}"
                
                success, errors = self.process_folder_recursively_with_archive_detection_move(
                    current_folder_path=new_current_path,
                    dest_base_folder=dest_base_folder,
                    source_folder_name=source_folder_name,
                    relative_path=new_relative_path
                )
                total_success += success
                total_errors += errors
            
            return total_success, total_errors
            
        except Exception as e:
            self.logger.error(f"Error in recursive MOVE processing for {current_folder_path}: {e}")
            return 0, 1
    
    def get_immediate_folder_contents(self, folder_path):
        """Get immediate files and subfolders (not recursive) for granular processing"""
        immediate_files = []
        immediate_subfolders = []
        
        try:
            # Encode folder path for URL
            if folder_path:
                encoded_path = quote(folder_path)
                url = f"https://graph.microsoft.com/v1.0/drives/{self.source_drive_id}/root:/{encoded_path}:/children?$top=1000"
            else:
                url = f"https://graph.microsoft.com/v1.0/drives/{self.source_drive_id}/root/children?$top=1000"
            
            while url:
                response = self.make_graph_request(url)
                
                if not response or response.status_code != 200:
                    break
                
                data = response.json()
                items = data.get('value', [])
                
                for item in items:
                    if 'folder' in item:
                        immediate_subfolders.append(item['name'])
                    else:
                        # This is a file - add to immediate files
                        item_path = f"{folder_path}/{item['name']}" if folder_path else item['name']
                        file_info = {
                            'name': item['name'],
                            'path': item_path,
                            'id': item['id'],
                            'size': item.get('size', 0),
                            'last_modified': item.get('lastModifiedDateTime'),
                            'created': item.get('createdDateTime')
                        }
                        immediate_files.append(file_info)
                
                # Check for next page
                url = data.get('@odata.nextLink')
            
            return immediate_files, immediate_subfolders
            
        except Exception as e:
            self.logger.error(f"Error getting immediate contents for {folder_path}: {e}")
            return [], []
    
    def check_if_all_files_archived(self, files):
        """Check if all files in the list are archived (older than threshold)"""
        if not files:
            return True  # Empty folder is considered archived
        
        recent_files = []
        for file_info in files:
            try:
                last_modified_str = file_info.get('last_modified')
                if last_modified_str:
                    # Parse SharePoint date format
                    last_modified = datetime.fromisoformat(last_modified_str.replace('Z', '+00:00'))
                    # Convert to naive datetime for comparison
                    last_modified_naive = last_modified.replace(tzinfo=None)
                    
                    if last_modified_naive > self.archive_threshold_date:
                        recent_files.append(file_info)
            except Exception as e:
                self.logger.warning(f"Error parsing date for {file_info.get('name', 'unknown')}: {e}")
        
        # If no recent files, all are archived
        return len(recent_files) == 0
    
    def move_subfolder_files(self, files, dest_path, subfolder_name):
        """Move files from a specific subfolder using TRUE MOVE operations"""
        success_count = 0
        error_count = 0
        completed = 0
        
        # Filter out already moved files
        unprocessed_files = [f for f in files if not self.is_file_moved(f)]
        
        if not unprocessed_files:
            self.logger.info(f"✅ All files in {subfolder_name} already moved")
            return len(files), 0
        
        self.logger.info(f"📂 Moving subfolder: {subfolder_name} - {len(unprocessed_files)} files")
        
        # Use ThreadPoolExecutor for parallel processing (conservative for moves)
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            # Submit all file move tasks
            future_to_file = {}
            for file_info in unprocessed_files:
                future = executor.submit(self.move_file_with_version_history, file_info, dest_path, subfolder_name)
                future_to_file[future] = file_info
            
            # Process completed tasks
            for future in as_completed(future_to_file):
                file_info = future_to_file[future]
                completed += 1
                
                try:
                    success = future.result()
                    if success:
                        success_count += 1
                        self.mark_file_moved(file_info, dest_path, "success")
                        # Update session statistics
                        self.session_stats["files_moved_this_session"] += 1
                        self.logger.info(f"✅ MOVED: {file_info['name']}")
                    else:
                        error_count += 1
                        self.mark_file_moved(file_info, dest_path, "error")
                        self.logger.warning(f"❌ MOVE FAILED: {file_info['name']}")
                    
                    # Save progress periodically
                    if completed % self.batch_save_frequency == 0:
                        self.save_all_progress()
                        self.print_progress_stats(completed, len(unprocessed_files), subfolder_name)
                    
                    # Delay for MOVE operations
                    time.sleep(self.move_delay)
                    
                except Exception as e:
                    error_count += 1
                    self.mark_file_moved(file_info, dest_path, "error")
                    self.logger.error(f"Exception moving {file_info['name']}: {e}")
        
        # Final save for this subfolder
        self.save_all_progress()
        
        self.logger.info(f"✅ Completed subfolder: {subfolder_name} - Moved: {success_count}, Errors: {error_count}")
        return success_count, error_count
    
    def mark_file_moved(self, file_info, dest_path, status):
        """Mark file as moved with session tracking"""
        file_id = file_info['id']
        with self.moved_files_lock:
            self.moved_files[file_id] = {
                'file_name': file_info['name'],
                'source_path': file_info['path'],
                'dest_path': dest_path,
                'status': status,
                'moved_timestamp': datetime.now().isoformat(),
                'session_id': self.session_id,
                'file_size': file_info.get('size', 0),
                'last_modified': file_info.get('last_modified')
            }
    
    def is_file_moved(self, file_info):
        """Check if file already moved"""
        file_id = file_info['id']
        return file_id in self.moved_files and self.moved_files[file_id].get('status') == 'success'
    
    def print_progress_stats(self, completed, total, folder_name):
        """Print progress statistics"""
        percent = (completed / total) * 100
        elapsed = time.time() - self.start_time
        rate = self.session_stats["files_moved_this_session"] / (elapsed / 3600) if elapsed > 0 else 0
        
        self.logger.info(f"📊 Progress: {completed}/{total} ({percent:.1f}%) | "
                        f"Rate: {rate:.1f} files/hour | "
                        f"Session total: {self.session_stats['files_moved_this_session']}")
    
    # Include all the archive detection methods from LEDFileCopier_Enhanced.py
    # (Same exact methods - archive detection logic doesn't change)
    
    def is_folder_archived(self, folder_path):
        """Check if a folder should be archived based on file modification dates"""
        try:
            # Get all files in the folder recursively
            files = self.get_files_in_folder_with_dates(folder_path)
            
            if not files:
                # Empty folder - consider as archived
                return True, "Empty folder"
            
            # Check if ALL files are older than threshold
            recent_files = []
            for file_info in files:
                try:
                    last_modified_str = file_info.get('last_modified')
                    if last_modified_str:
                        # Parse SharePoint date format
                        last_modified = datetime.fromisoformat(last_modified_str.replace('Z', '+00:00'))
                        # Convert to naive datetime for comparison
                        last_modified_naive = last_modified.replace(tzinfo=None)
                        
                        if last_modified_naive > self.archive_threshold_date:
                            recent_files.append({
                                'name': file_info['name'],
                                'last_modified': last_modified_str,
                                'days_old': (datetime.now() - last_modified_naive).days
                            })
                except Exception as e:
                    self.logger.warning(f"Error parsing date for {file_info.get('name', 'unknown')}: {e}")
            
            if not recent_files:
                # All files are older than 2 years
                oldest_file = min(files, key=lambda x: x.get('last_modified', ''))
                youngest_file = max(files, key=lambda x: x.get('last_modified', ''))
                return True, f"All {len(files)} files older than 2 years. Oldest: {oldest_file.get('last_modified', 'unknown')}, Youngest: {youngest_file.get('last_modified', 'unknown')}"
            else:
                return False, f"{len(recent_files)} files modified within 2 years (out of {len(files)} total)"
                
        except Exception as e:
            self.logger.error(f"Error checking archive status for {folder_path}: {e}")
            return False, f"Error checking archive status: {e}"
    
    def get_files_in_folder_with_dates(self, folder_path):
        """Get all files recursively with modification dates"""
        files = []
        return self._get_files_recursive_with_dates(folder_path, files)
    
    def _get_files_recursive_with_dates(self, current_path, files):
        """Recursively get all files with dates"""
        if current_path:
            url = f"https://graph.microsoft.com/v1.0/drives/{self.source_drive_id}/root:/{current_path}:/children?$top=1000"
        else:
            url = f"https://graph.microsoft.com/v1.0/drives/{self.source_drive_id}/root/children?$top=1000"
        
        while url:
            response = self.make_graph_request(url)
            
            if not response or response.status_code != 200:
                break
            
            data = response.json()
            items = data.get('value', [])
            
            for item in items:
                if 'folder' in item:
                    # Recursively get files from this folder
                    folder_path = f"{current_path}/{item['name']}" if current_path else item['name']
                    self._get_files_recursive_with_dates(folder_path, files)
                else:
                    # This is a file
                    item_path = f"{current_path}/{item['name']}" if current_path else item['name']
                    file_info = {
                        'name': item['name'],
                        'path': item_path,
                        'id': item['id'],
                        'size': item.get('size', 0),
                        'last_modified': item.get('lastModifiedDateTime'),
                        'created': item.get('createdDateTime'),
                        'relative_path': item_path.replace(f"{current_path.split('/')[0]}/", "") if '/' in current_path else item['name']
                    }
                    files.append(file_info)
            
            # Check for next page
            url = data.get('@odata.nextLink')
        
        return files
    
    # Load/Save methods (adapted for move operations)
    def load_moved_files(self):
        """Load moved files database"""
        if os.path.exists(self.moved_files_db):
            try:
                with open(self.moved_files_db, 'r') as f:
                    return json.load(f)
            except Exception as e:
                self.logger.error(f"Error loading moved files: {e}")
                return {}
        return {}
    
    def load_failed_moves(self):
        """Load failed moves database"""
        if os.path.exists(self.failed_moves_report):
            try:
                with open(self.failed_moves_report, 'r') as f:
                    return json.load(f)
            except Exception as e:
                self.logger.error(f"Error loading failed moves: {e}")
                return {"failed_moves": [], "summary": {}}
        return {"failed_moves": [], "summary": {}}
    
    def load_archived_folders(self):
        """Load archived folders database"""
        if os.path.exists(self.archived_folders_log):
            try:
                with open(self.archived_folders_log, 'r') as f:
                    return json.load(f)
            except Exception as e:
                self.logger.error(f"Error loading archived folders: {e}")
                return {"archived_folders": [], "summary": {}}
        return {"archived_folders": [], "summary": {}}
    
    def load_move_mappings(self):
        """Load move mappings database"""
        if os.path.exists(self.move_mapping_log):
            try:
                with open(self.move_mapping_log, 'r') as f:
                    return json.load(f)
            except Exception as e:
                self.logger.error(f"Error loading move mappings: {e}")
                return {"move_mappings": [], "summary": {}}
        return {"move_mappings": [], "summary": {}}
    
    def log_move_mapping(self, source_path, dest_path, action, status, additional_info=None):
        """Log file move mapping for audit trail with session tracking"""
        mapping_entry = {
            "timestamp": datetime.now().isoformat(),
            "session_id": self.session_id,
            "source_path": source_path,
            "destination_path": dest_path,
            "action": action,
            "status": status
        }
        
        if additional_info:
            mapping_entry["additional_info"] = additional_info
        
        with self.move_mappings_lock:
            if "move_mappings" not in self.move_mappings:
                self.move_mappings["move_mappings"] = []
            
            self.move_mappings["move_mappings"].append(mapping_entry)
            
            # Update summary
            self.move_mappings["summary"] = {
                "total_mappings": len(self.move_mappings["move_mappings"]),
                "last_updated": datetime.now().isoformat(),
                "current_session_id": self.session_id
            }
    
    def save_all_progress(self):
        """Save all progress files"""
        self.save_moved_files()
        self.save_failed_moves()
        self.save_archived_folders()
        self.save_move_mappings()
    
    def save_moved_files(self):
        """Save moved files database"""
        try:
            temp_file = f"{self.moved_files_db}.tmp"
            with open(temp_file, 'w') as f:
                json.dump(self.moved_files, f, indent=2)
            os.rename(temp_file, self.moved_files_db)
        except Exception as e:
            self.logger.error(f"Error saving moved files: {e}")
    
    def save_failed_moves(self):
        """Save failed moves database"""
        try:
            with open(self.failed_moves_report, 'w') as f:
                json.dump(self.failed_moves, f, indent=2)
        except Exception as e:
            self.logger.error(f"Error saving failed moves: {e}")
    
    def save_archived_folders(self):
        """Save archived folders database"""
        try:
            with open(self.archived_folders_log, 'w') as f:
                json.dump(self.archived_folders, f, indent=2)
        except Exception as e:
            self.logger.error(f"Error saving archived folders: {e}")
    
    def save_move_mappings(self):
        """Save move mappings database"""
        try:
            with open(self.move_mapping_log, 'w') as f:
                json.dump(self.move_mappings, f, indent=2)
        except Exception as e:
            self.logger.error(f"Error saving move mappings: {e}")

def main():
    """Main function for scheduled daily migration"""
    
    # CONFIGURE YOUR DAILY FOLDERS HERE
    # Replace this list with folders you want to migrate today
    daily_folders_to_process = [
        # Example configuration - REPLACE WITH YOUR DAILY FOLDERS:
        # "ConnectWIse Today",
        # "Sales",
        # "Marketing"
    ]
    
    print("📅 LED File Mover - SCHEDULED DAILY MIGRATION")
    print("=" * 80)
    print("🧪 QA TESTING MODE - Using Test Site /sites/LEDDocuments")
    print("🔒 Production /sites/Documents is OFF-LIMITS during QA phase")
    print("=" * 80)
    print("🆕 Enhanced Features:")
    print("   ✅ Scheduled daily folder processing")
    print("   ✅ Enhanced multi-threading (all workers focus on same folder)")
    print("   ✅ Session tracking with timestamps")
    print("   ✅ TRUE MOVE operations (preserves version history)")
    print("   ✅ Archive detection (same as Enhanced version)")
    print("   ✅ Atomic transactions (safer than copy+delete)")
    print("   ✅ Comprehensive audit trail with session IDs")
    print("=" * 80)
    
    if not daily_folders_to_process:
        print("❌ No folders scheduled for migration today")
        print("💡 Edit the daily_folders_to_process list in main() function")
        print("📝 Example: daily_folders_to_process = ['ConnectWIse Today', 'Sales']")
        return
    
    print(f"\n📅 TODAY'S MIGRATION SCHEDULE:")
    for i, folder in enumerate(daily_folders_to_process, 1):
        print(f"   {i}. {folder}")
    print("=" * 80)
    
    # Initialize the mover with scheduled folders
    mapping_file = "/Users/dhilen/Documents/Coding/LED File Copier/Onedrive folder mapping - DW Approved.xlsx"
    
    print(f"\n🔧 Initializing LED File Mover for scheduled migration...")
    mover = LEDFileMover(mapping_file, daily_folders_to_process)
    
    # Authenticate
    if not mover.get_access_token():
        print("❌ Failed to authenticate")
        return
    
    print("✅ Authentication successful")
    
    # Initialize drives
    if not mover.initialize_drives():
        print("❌ Failed to initialize SharePoint drives")
        return
    
    print("✅ SharePoint drives initialized")
    
    print(f"\n🔍 Archive Detection Settings:")
    print(f"   - Archive threshold: {mover.archive_threshold_days} days ({mover.archive_threshold_days/365:.1f} years)")
    print(f"   - Archive date cutoff: {mover.archive_threshold_date.strftime('%Y-%m-%d')}")
    
    print(f"\n⚙️ Enhanced Threading Settings:")
    print(f"   - Parallel workers: {mover.max_workers} (focusing on same folder)")
    print(f"   - Move delay: {mover.move_delay}s")
    print(f"   - Progress save frequency: every {mover.batch_save_frequency} files")
    print(f"   - TRUE MOVE operations: ✅ (preserves version history)")
    print(f"   - Session ID: {mover.session_id}")
    
    # Check existing progress
    existing_moved = len(mover.moved_files)
    existing_archived = len(mover.archived_folders.get("archived_folders", []))
    
    if existing_moved > 0 or existing_archived > 0:
        print(f"\n📊 Existing Progress Found:")
        print(f"   - Files already moved: {existing_moved:,}")
        print(f"   - Folders already archived: {existing_archived:,}")
        print(f"   - Will resume/continue with scheduled folders...")
    
    print(f"\n📁 Logging Files:")
    print(f"   - Operation log: led_file_mover.log")
    print(f"   - Session log: migration_session_{mover.session_id}.json")
    print(f"   - Progress tracking: moved_files_database.json")
    print(f"   - Move mappings: move_audit_trail.json")
    print(f"   - Archived folders: archived_folders_registry.json")
    print(f"   - Failed moves: failed_moves_report.json")
    
    print(f"\n🚀 Starting scheduled migration of {len(daily_folders_to_process)} folders...")
    print("   💡 Enhanced features active:")
    print("   📅 Scheduled daily processing")
    print("   🧵 Enhanced multi-threading")
    print("   🔄 TRUE MOVE operations with version history")
    print("   🗃️ Archive detection for old folders")
    print("   📋 Session-tracked audit trail")
    print("   💾 Safe to stop anytime (Ctrl+C) and resume later")
    
    try:
        # Start the scheduled migration
        success_count, error_count = mover.move_all_mapped_folders_with_archive_detection()
        
        print(f"\n🎉 SCHEDULED MIGRATION COMPLETED SUCCESSFULLY")
        print(f"📅 Session: {mover.session_id}")
        
    except KeyboardInterrupt:
        print(f"\n⏸️ Migration stopped by user")
        print(f"💾 Progress saved - you can resume later")
        mover.save_all_progress()
        mover.log_session_end()
        
    except Exception as e:
        print(f"\n❌ Migration failed with error: {e}")
        mover.logger.error(f"Migration failed: {e}")
        mover.save_all_progress()
        mover.log_session_end()

if __name__ == "__main__":
    main()