main.py

#!/usr/bin/env python3
"""
Google Docs BFS Exporter

This script performs a breadth-first search (BFS) through Google Docs by following links,
and exports each document as either .docx or markdown.

Usage:
    1. First run: python main.py --setup
       This will guide you through OAuth setup and save credentials
    
    2. Export docs: python main.py --seed-id YOUR_DOC_ID --format md
       Starts BFS from the seed document and exports all linked docs
"""

import os
import re
import csv
import json
import pickle
import argparse
from collections import deque
from pathlib import Path
from typing import Set, Optional, Dict, List
from urllib.parse import urlparse, parse_qs

from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaInMemoryUpload

# If modifying these scopes, delete the file token.pickle.
# Note: drive.readonly is sufficient for requesting access (it allows reading permissions)
# drive.file is needed for uploading to Drive (only accesses files created by this app)
SCOPES = ['https://www.googleapis.com/auth/documents.readonly',
          'https://www.googleapis.com/auth/drive.readonly',
          'https://www.googleapis.com/auth/drive.file']

# Paths
TOKEN_PATH = 'token.pickle'
CREDENTIALS_PATH = 'credentials.json'
OUTPUT_DIR = Path('exported_docs')
INDEX_CSV = OUTPUT_DIR / 'index.csv'


class GoogleDocsCrawler:
    """Crawls Google Docs via BFS and exports them."""
    
    def __init__(self, export_format: str = 'md', localize_links: bool = False, request_access: bool = False, 
                 drive_folder_id: Optional[str] = None):
        """
        Initialize the crawler.
        
        Args:
            export_format: Either 'md' for markdown or 'docx' for Word format
            localize_links: If True, convert Google Docs links to local markdown file links
            request_access: If True, automatically request access when encountering permission errors
            drive_folder_id: If provided, save copies to this Google Drive folder instead of local export
        """
        self.export_format = export_format.lower()
        if self.export_format not in ['md', 'docx']:
            raise ValueError("export_format must be 'md' or 'docx'")
        
        self.localize_links = localize_links
        self.request_access = request_access
        self.drive_folder_id = drive_folder_id
        self.creds = self._get_credentials()
        self.docs_service = build('docs', 'v1', credentials=self.creds)
        self.drive_service = build('drive', 'v3', credentials=self.creds)
        
        # Get user email for access requests
        self.user_email = None
        if self.request_access:
            self.user_email = self._get_user_email()
        
        # Verify Drive folder if provided
        if self.drive_folder_id:
            self._verify_drive_folder()
        
        # Track exported documents: doc_id -> (title, filename_or_drive_id)
        self.exported_docs: Dict[str, tuple[str, str]] = {}
        
        # Create output directory (still needed for index CSV and temp files)
        OUTPUT_DIR.mkdir(exist_ok=True)
        
    def _get_credentials(self) -> Credentials:
        """Get valid user credentials from storage or run OAuth flow."""
        creds = None
        
        # Load existing credentials
        if os.path.exists(TOKEN_PATH):
            with open(TOKEN_PATH, 'rb') as token:
                creds = pickle.load(token)
        
        # If no valid credentials, let user log in
        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                print("Refreshing access token...")
                creds.refresh(Request())
            else:
                if not os.path.exists(CREDENTIALS_PATH):
                    raise FileNotFoundError(
                        f"Missing {CREDENTIALS_PATH}. Run with --setup first."
                    )
                print("Starting OAuth flow...")
                flow = InstalledAppFlow.from_client_secrets_file(
                    CREDENTIALS_PATH, SCOPES)
                creds = flow.run_local_server(port=0)
            
            # Save credentials for next run
            with open(TOKEN_PATH, 'wb') as token:
                pickle.dump(creds, token)
            print("Credentials saved!")
        
        return creds
    
    def _get_user_email(self) -> Optional[str]:
        """Get the authenticated user's email address."""
        try:
            about = self.drive_service.about().get(fields='user').execute()
            email = about.get('user', {}).get('emailAddress')
            if email:
                print(f"📧 Using email for access requests: {email}")
            return email
        except HttpError as e:
            print(f"⚠ Could not get user email: {e}")
            return None
    
    def _verify_drive_folder(self):
        """Verify that the Drive folder exists and is accessible."""
        try:
            folder = self.drive_service.files().get(
                fileId=self.drive_folder_id,
                fields='id,name,mimeType'
            ).execute()
            
            if folder.get('mimeType') != 'application/vnd.google-apps.folder':
                raise ValueError(f"ID {self.drive_folder_id} is not a folder")
            
            print(f"📁 Target Drive folder: {folder.get('name')} ({self.drive_folder_id})")
            
        except HttpError as e:
            if e.resp.status == 404:
                raise ValueError(f"Drive folder {self.drive_folder_id} not found")
            elif e.resp.status == 403:
                raise ValueError(f"No access to Drive folder {self.drive_folder_id}")
            else:
                raise ValueError(f"Error accessing Drive folder: {e}")
    
    def extract_doc_id_from_url(self, url: str) -> Optional[str]:
        """
        Extract document ID from a Google Docs URL.
        
        Args:
            url: Google Docs URL
            
        Returns:
            Document ID or None if not a valid Google Docs URL
        """
        # Pattern: https://docs.google.com/document/d/{DOC_ID}/...
        match = re.search(r'/document/d/([a-zA-Z0-9-_]+)', url)
        if match:
            return match.group(1)
        return None
    
    def request_access_to_file(self, doc_id: str, user_email: str) -> bool:
        """
        Request access to a Google Doc by creating a pending permission request.
        
        Args:
            doc_id: Google Docs document ID
            user_email: Email address of the user requesting access
            
        Returns:
            True if request was sent successfully, False otherwise
        """
        try:
            # Create a permission request for the user
            # Setting role to 'reader' and sendNotificationEmail=True triggers an access request
            # The permission will be in 'pending' state until approved by the owner
            self.drive_service.permissions().create(
                fileId=doc_id,
                body={
                    'type': 'user',
                    'role': 'reader',
                    'emailAddress': user_email
                },
                sendNotificationEmail=True,
                emailMessage='Automated access request from Google Docs BFS Exporter',
                fields='id'
            ).execute()
            return True
        except HttpError as e:
            # This will fail with 403 if we can't even request access
            # (e.g., sharing is completely disabled, or we already have access)
            error_reason = 'unknown'
            try:
                error_content = json.loads(e.content.decode('utf-8'))
                if 'error' in error_content and 'errors' in error_content['error']:
                    error_reason = error_content['error']['errors'][0].get('reason', 'unknown')
            except:
                pass
            
            if error_reason == 'sharingRateLimitExceeded':
                print(f"  ⚠ Rate limit exceeded for access requests")
            else:
                print(f"  ⚠ Could not send access request: {error_reason}")
            return False
    
    def get_document(self, doc_id: str) -> dict:
        """
        Fetch a Google Doc by ID.
        
        Args:
            doc_id: Google Docs document ID
            
        Returns:
            Document resource, or None if error
        """
        try:
            return self.docs_service.documents().get(documentId=doc_id).execute()
        except HttpError as e:
            # Check if this is a permission error (403)
            if e.resp.status == 403 and self.request_access and self.user_email:
                print(f"  ⚠ Permission denied for document {doc_id}")
                print(f"  📧 Requesting access...")
                if self.request_access_to_file(doc_id, self.user_email):
                    print(f"  ✓ Access request sent! Skipping for now (re-run after access is granted)")
                else:
                    print(f"  ✗ Failed to request access")
            else:
                print(f"Error fetching document {doc_id}: {e}")
            return None
    
    def extract_links_from_doc(self, doc: dict) -> Set[str]:
        """
        Extract all Google Docs links from a document.
        
        Args:
            doc: Document resource from Google Docs API
            
        Returns:
            Set of document IDs found in links
        """
        doc_ids = set()
        
        def process_element(element):
            """Recursively process document elements to find links."""
            # Check for links in text runs
            if 'textRun' in element:
                text_run = element['textRun']
                if 'textStyle' in text_run and 'link' in text_run['textStyle']:
                    url = text_run['textStyle']['link'].get('url', '')
                    doc_id = self.extract_doc_id_from_url(url)
                    if doc_id:
                        doc_ids.add(doc_id)
            
            # Recursively process nested elements
            for key in ['paragraph', 'table', 'tableRow', 'tableCell']:
                if key in element:
                    for sub_element in element[key].get('elements', []):
                        process_element(sub_element)
        
        # Process all content in the document
        content = doc.get('body', {}).get('content', [])
        for element in content:
            process_element(element)
        
        return doc_ids
    
    def format_text_with_style(self, text: str, style: dict) -> str:
        """
        Apply markdown formatting to text based on style.
        
        Args:
            text: Text content
            style: Text style dictionary from Google Docs API
            
        Returns:
            Formatted text with markdown syntax
        """
        if not text or not style:
            return text
        
        # Apply formatting in order: bold, italic, strikethrough, code
        formatted = text
        
        # Bold and Italic (handle combination first)
        is_bold = style.get('bold', False)
        is_italic = style.get('italic', False)
        
        if is_bold and is_italic:
            formatted = f"***{formatted}***"
        elif is_bold:
            formatted = f"**{formatted}**"
        elif is_italic:
            formatted = f"*{formatted}*"
        
        # Strikethrough (applied after bold/italic)
        if style.get('strikethrough'):
            formatted = f"~~{formatted}~~"
        
        # Code (monospace font)
        if style.get('weightedFontFamily', {}).get('fontFamily') == 'Consolas' or \
           style.get('weightedFontFamily', {}).get('fontFamily') == 'Courier New':
            formatted = f"`{formatted}`"
        
        return formatted
    
    def convert_link_to_local(self, url: str, text: str) -> str:
        """
        Convert a Google Docs URL to a local markdown link if possible.
        
        Args:
            url: Original URL
            text: Link text
            
        Returns:
            Markdown link syntax
        """
        if not self.localize_links or self.export_format != 'md':
            return f"[{text}]({url})"
        
        # Check if it's a Google Docs link
        doc_id = self.extract_doc_id_from_url(url)
        if doc_id and doc_id in self.exported_docs:
            # Convert to local link
            _, local_filename = self.exported_docs[doc_id]
            return f"[{text}]({local_filename})"
        
        # Not a docs link or not exported yet, keep original
        return f"[{text}]({url})"
    
    def upload_to_drive(self, content: bytes, filename: str, mime_type: str) -> str:
        """
        Upload a file to the specified Drive folder.
        
        Args:
            content: File content as bytes
            filename: Name for the file in Drive
            mime_type: MIME type of the file
            
        Returns:
            File ID of the uploaded file
        """
        file_metadata = {
            'name': filename,
            'parents': [self.drive_folder_id]
        }
        
        media = MediaInMemoryUpload(content, mimetype=mime_type, resumable=True)
        
        file = self.drive_service.files().create(
            body=file_metadata,
            media_body=media,
            fields='id,name,webViewLink'
        ).execute()
        
        return file.get('id')
    
    def copy_doc_to_drive(self, doc_id: str, new_title: str) -> Optional[str]:
        """
        Copy a Google Doc to the specified Drive folder.
        
        Args:
            doc_id: Source document ID
            new_title: Title for the copied document
            
        Returns:
            File ID of the copied document, or None if copy failed
        """
        try:
            file_metadata = {
                'name': new_title,
                'parents': [self.drive_folder_id]
            }
            
            copied_file = self.drive_service.files().copy(
                fileId=doc_id,
                body=file_metadata,
                fields='id,name,webViewLink'
            ).execute()
            
            return copied_file.get('id')
        except HttpError as e:
            # 404 means the file isn't accessible via Drive API (might be in a shared drive, etc.)
            print(f"  ✗ Cannot copy via Drive API (error {e.resp.status}): {e.error_details if hasattr(e, 'error_details') else 'file not accessible'}")
            return None
    
    def export_as_markdown(self, doc: dict, output_path: Path, title: str):
        """
        Export document as markdown with improved formatting support.
        
        Args:
            doc: Document resource from Google Docs API
            output_path: Path to save markdown file (or None if uploading to Drive)
            title: Document title (used for Drive uploads)
        """
        lines = []
        title = doc.get('title', 'Untitled')
        lines.append(f"# {title}\n\n")
        
        content = doc.get('body', {}).get('content', [])
        lists = doc.get('lists', {})  # List definitions
        
        for element in content:
            if 'paragraph' in element:
                paragraph = element['paragraph']
                text_parts = []
                
                # Get paragraph style
                para_style = paragraph.get('paragraphStyle', {})
                named_style = para_style.get('namedStyleType', '')
                
                # Check if this is a list item
                bullet = paragraph.get('bullet')
                indent_level = 0
                list_prefix = ''
                
                if bullet:
                    # Get indent level
                    indent = para_style.get('indentFirstLine', {})
                    indent_start = para_style.get('indentStart', {})
                    
                    # Estimate indent level (Google Docs uses points, roughly 36pt per level)
                    if indent_start:
                        magnitude = indent_start.get('magnitude', 0)
                        indent_level = max(0, int(magnitude / 36))
                    
                    # Determine if ordered or unordered
                    list_id = bullet.get('listId')
                    nesting_level = bullet.get('nestingLevel', 0)
                    
                    # Get list properties
                    is_ordered = False
                    if list_id and list_id in lists:
                        list_props = lists[list_id].get('listProperties', {})
                        nesting_props = list_props.get('nestingLevels', [])
                        if nesting_level < len(nesting_props):
                            glyph_type = nesting_props[nesting_level].get('glyphType', '')
                            # Ordered lists use DECIMAL, ALPHA, etc.
                            is_ordered = glyph_type not in ['GLYPH_TYPE_UNSPECIFIED', '']
                            # Common ordered types
                            if any(x in glyph_type for x in ['DECIMAL', 'ALPHA', 'ROMAN']):
                                is_ordered = True
                    
                    # Create list prefix
                    indent_str = '  ' * indent_level
                    if is_ordered:
                        list_prefix = f"{indent_str}1. "
                    else:
                        list_prefix = f"{indent_str}- "
                
                # Process text elements
                for elem in paragraph.get('elements', []):
                    if 'textRun' in elem:
                        text_run = elem['textRun']
                        content_text = text_run.get('content', '')
                        
                        # Apply text formatting
                        style = text_run.get('textStyle', {})
                        
                        # Handle links specially
                        if 'link' in style:
                            url = style['link'].get('url', '')
                            # Strip newlines from link text
                            link_text = content_text.strip()
                            if link_text:
                                content_text = self.convert_link_to_local(url, link_text)
                                # Add back newline if it was there
                                if text_run.get('content', '').endswith('\n'):
                                    content_text += '\n'
                        else:
                            # Apply other formatting
                            content_text = self.format_text_with_style(content_text, style)
                        
                        text_parts.append(content_text)
                
                line = ''.join(text_parts)
                
                # Apply heading styles
                if named_style.startswith('HEADING_'):
                    level = named_style.split('_')[1]
                    if level.isdigit():
                        line = '#' * int(level) + ' ' + line.strip() + '\n'
                elif list_prefix:
                    # This is a list item
                    line = list_prefix + line.lstrip()
                
                lines.append(line)
            
            elif 'table' in element:
                # Basic table support
                table = element['table']
                lines.append('\n')  # Add spacing before table
                
                for row_idx, table_row in enumerate(table.get('tableRows', [])):
                    cells = []
                    for table_cell in table_row.get('tableCells', []):
                        cell_content = []
                        for cell_element in table_cell.get('content', []):
                            if 'paragraph' in cell_element:
                                para = cell_element['paragraph']
                                for elem in para.get('elements', []):
                                    if 'textRun' in elem:
                                        text = elem['textRun'].get('content', '').strip()
                                        cell_content.append(text)
                        cells.append(' '.join(cell_content))
                    
                    # Create table row
                    lines.append('| ' + ' | '.join(cells) + ' |\n')
                    
                    # Add header separator after first row
                    if row_idx == 0:
                        lines.append('| ' + ' | '.join(['---'] * len(cells)) + ' |\n')
                
                lines.append('\n')  # Add spacing after table
        
        markdown_content = ''.join(lines)
        
        if self.drive_folder_id:
            # Upload to Drive as a text file with original title
            try:
                content_bytes = markdown_content.encode('utf-8')
                filename = f"{self.sanitize_filename(title)}.md"
                file_id = self.upload_to_drive(content_bytes, filename, 'text/markdown')
                print(f"✓ Uploaded markdown to Drive: {title} (ID: {file_id})")
                return file_id
            except HttpError as e:
                print(f"  ✗ Error uploading markdown to Drive: {e}")
                # Save locally as fallback
                output_path.write_text(markdown_content, encoding='utf-8')
                print(f"  ⚠ Saved locally as fallback: {output_path}")
                return None
        else:
            # Save locally
            output_path.write_text(markdown_content, encoding='utf-8')
            print(f"✓ Exported as markdown: {output_path}")
            return None
    
    def export_as_docx(self, doc_id: str, output_path: Path, title: str):
        """
        Export document as .docx using Google Drive API.
        
        Args:
            doc_id: Google Docs document ID
            output_path: Path to save .docx file (or None if copying to Drive)
            title: Document title (used for Drive uploads)
            
        Returns:
            File ID if uploaded to Drive, None otherwise
        """
        try:
            if self.drive_folder_id:
                # Try to copy the Google Doc directly to the target folder first
                # This preserves it as a Google Doc (not .docx)
                file_id = self.copy_doc_to_drive(doc_id, title)
                if file_id:
                    print(f"✓ Copied to Drive folder: {title} (ID: {file_id})")
                    return file_id
                else:
                    # Fallback: Export as .docx and upload
                    print(f"  ⤷ Falling back to .docx export and upload...")
                    request = self.drive_service.files().export_media(
                        fileId=doc_id,
                        mimeType='application/vnd.openxmlformats-officedocument.wordprocessingml.document'
                    )
                    content = request.execute()
                    filename = f"{self.sanitize_filename(title)}.docx"
                    file_id = self.upload_to_drive(content, filename, 
                        'application/vnd.openxmlformats-officedocument.wordprocessingml.document')
                    print(f"  ✓ Uploaded .docx to Drive: {title} (ID: {file_id})")
                    return file_id
            else:
                # Export as DOCX and save locally
                request = self.drive_service.files().export_media(
                    fileId=doc_id,
                    mimeType='application/vnd.openxmlformats-officedocument.wordprocessingml.document'
                )
                
                content = request.execute()
                output_path.write_bytes(content)
                print(f"✓ Exported as docx: {output_path}")
                return None
            
        except HttpError as e:
            print(f"  ✗ Error exporting document {doc_id} as DOCX: {e}")
            return None
    
    def sanitize_filename(self, title: str) -> str:
        """
        Sanitize document title for use as filename.
        
        Args:
            title: Document title
            
        Returns:
            Safe filename
        """
        # Remove invalid characters
        safe = re.sub(r'[<>:"/\\|?*]', '', title)
        # Limit length
        safe = safe[:100]
        return safe or 'untitled'
    
    def save_index_csv(self):
        """Save an index CSV with document IDs, titles, and local paths or Drive file IDs."""
        with open(INDEX_CSV, 'w', newline='', encoding='utf-8') as f:
            writer = csv.writer(f)
            
            if self.drive_folder_id:
                writer.writerow(['Source Doc ID', 'Title', 'Drive File ID', 'Source URL', 'Drive URL'])
                
                for doc_id, (title, drive_file_id) in sorted(self.exported_docs.items()):
                    source_url = f"https://docs.google.com/document/d/{doc_id}/edit"
                    
                    # Construct Drive URL based on file type
                    if self.export_format == 'md':
                        drive_url = f"https://drive.google.com/file/d/{drive_file_id}/view"
                    else:  # docx format (actually still a Google Doc)
                        drive_url = f"https://docs.google.com/document/d/{drive_file_id}/edit"
                    
                    writer.writerow([doc_id, title, drive_file_id, source_url, drive_url])
            else:
                writer.writerow(['Document ID', 'Title', 'Local Filename', 'Google Docs URL'])
                
                for doc_id, (title, filename) in sorted(self.exported_docs.items()):
                    url = f"https://docs.google.com/document/d/{doc_id}/edit"
                    writer.writerow([doc_id, title, filename, url])
        
        print(f"\n📋 Index saved to: {INDEX_CSV.absolute()}")
    
    def crawl_and_export(self, seed_doc_id: str, max_docs: int = 100):
        """
        Perform BFS crawl starting from seed document.
        
        Args:
            seed_doc_id: Document ID to start crawling from
            max_docs: Maximum number of documents to export (safety limit)
        """
        visited: Set[str] = set()
        queue = deque([seed_doc_id])
        count = 0
        
        print(f"\n🚀 Starting BFS crawl from document: {seed_doc_id}")
        print(f"Export format: {self.export_format.upper()}")
        if self.drive_folder_id:
            print(f"Export target: Google Drive folder {self.drive_folder_id}")
        else:
            print(f"Export target: Local directory {OUTPUT_DIR.absolute()}")
        print(f"Localize links: {'Yes' if self.localize_links else 'No'}\n")
        
        # First pass: export all documents
        while queue and count < max_docs:
            doc_id = queue.popleft()
            
            if doc_id in visited:
                continue
            
            visited.add(doc_id)
            count += 1
            
            print(f"\n[{count}/{max_docs}] Processing: {doc_id}")
            
            # Fetch document
            doc = self.get_document(doc_id)
            if not doc:
                print(f"  ⚠ Skipping (couldn't fetch document)")
                continue
            
            title = doc.get('title', 'Untitled')
            print(f"  Title: {title}")
            
            # Export document
            safe_filename = self.sanitize_filename(title)
            extension = self.export_format
            filename = f"{safe_filename}_{doc_id}.{extension}"
            output_path = OUTPUT_DIR / filename
            
            # Export and get file ID (for Drive) or filename (for local)
            if self.export_format == 'md':
                file_id_or_name = self.export_as_markdown(doc, output_path, title)
            else:  # docx
                file_id_or_name = self.export_as_docx(doc_id, output_path, title)
            
            # Track this export (Drive file ID or local filename)
            if self.drive_folder_id:
                if file_id_or_name:
                    self.exported_docs[doc_id] = (title, file_id_or_name)
                else:
                    print(f"  ⚠ Export failed - file not uploaded to Drive")
            else:
                self.exported_docs[doc_id] = (title, filename)
            
            # Extract links and add to queue
            linked_doc_ids = self.extract_links_from_doc(doc)
            new_docs = linked_doc_ids - visited
            
            if new_docs:
                print(f"  Found {len(new_docs)} new linked document(s)")
                queue.extend(new_docs)
            else:
                print(f"  No new linked documents found")
        
        # If localizing links, do a second pass to update markdown files
        if self.localize_links and self.export_format == 'md':
            if self.drive_folder_id:
                print("\n🔗 Second pass: Localizing links to Drive URLs in markdown files...")
                self.localize_markdown_links_drive()
            else:
                print("\n🔗 Second pass: Localizing links in markdown files...")
                self.localize_markdown_links()
        
        # Note: For docx format with Drive, links remain as-is pointing to original docs
        # Google Docs API doesn't easily support bulk link replacement in copied docs
        if self.localize_links and self.export_format == 'docx' and self.drive_folder_id:
            print("\n⚠ Note: Link localization for docx format in Drive is not supported.")
            print("   Links in copied docs will still point to original documents.")
        
        # Save index CSV
        self.save_index_csv()
        
        # Report statistics
        successful_exports = len(self.exported_docs)
        print(f"\n✅ Crawl complete!")
        print(f"   Documents processed: {count}")
        print(f"   Successfully exported: {successful_exports}")
        if self.drive_folder_id:
            print(f"   Target: Google Drive folder {self.drive_folder_id}")
        else:
            print(f"   Target: {OUTPUT_DIR.absolute()}")
        
        if successful_exports < count:
            failed = count - successful_exports
            print(f"   ⚠ Failed exports: {failed}")
        
        if queue:
            print(f"\n⚠ Stopped at max_docs limit ({max_docs}). {len(queue)} documents remaining in queue.")
    
    def localize_markdown_links(self):
        """
        Second pass: Replace Google Docs links with local markdown links.
        This is done after all documents are exported so we know all filenames.
        """
        for doc_id, (title, filename) in self.exported_docs.items():
            filepath = OUTPUT_DIR / filename
            
            if not filepath.exists():
                continue
            
            content = filepath.read_text(encoding='utf-8')
            modified = False
            
            # Find all markdown links
            link_pattern = r'\[([^\]]+)\]\((https://docs\.google\.com/document/d/([a-zA-Z0-9-_]+)[^\)]*)\)'
            
            def replace_link(match):
                nonlocal modified
                link_text = match.group(1)
                full_url = match.group(2)
                linked_doc_id = match.group(3)
                
                # If we exported this document, replace with local link
                if linked_doc_id in self.exported_docs:
                    modified = True
                    _, linked_filename = self.exported_docs[linked_doc_id]
                    return f"[{link_text}]({linked_filename})"
                
                # Keep original
                return match.group(0)
            
            new_content = re.sub(link_pattern, replace_link, content)
            
            if modified:
                filepath.write_text(new_content, encoding='utf-8')
                print(f"  ✓ Localized links in: {filename}")
    
    def localize_markdown_links_drive(self):
        """
        Second pass: Replace Google Docs links with Drive URLs in uploaded markdown files.
        Downloads from Drive, updates links, and re-uploads.
        """
        for doc_id, (title, drive_file_id) in self.exported_docs.items():
            try:
                # Download the markdown file from Drive
                request = self.drive_service.files().get_media(fileId=drive_file_id)
                content = request.execute().decode('utf-8')
                
                modified = False
                
                # Find all markdown links
                link_pattern = r'\[([^\]]+)\]\((https://docs\.google\.com/document/d/([a-zA-Z0-9-_]+)[^\)]*)\)'
                
                def replace_link(match):
                    nonlocal modified
                    link_text = match.group(1)
                    full_url = match.group(2)
                    linked_doc_id = match.group(3)
                    
                    # If we exported this document, replace with Drive URL
                    if linked_doc_id in self.exported_docs:
                        modified = True
                        _, linked_drive_file_id = self.exported_docs[linked_doc_id]
                        # Use Drive file view URL
                        drive_url = f"https://drive.google.com/file/d/{linked_drive_file_id}/view"
                        return f"[{link_text}]({drive_url})"
                    
                    # Keep original
                    return match.group(0)
                
                new_content = re.sub(link_pattern, replace_link, content)
                
                if modified:
                    # Update the file in Drive
                    media = MediaInMemoryUpload(
                        new_content.encode('utf-8'),
                        mimetype='text/markdown',
                        resumable=True
                    )
                    self.drive_service.files().update(
                        fileId=drive_file_id,
                        media_body=media
                    ).execute()
                    print(f"  ✓ Localized links in Drive: {title}")
                    
            except HttpError as e:
                print(f"  ⚠ Error updating links in {title}: {e}")


def setup_oauth():
    """Guide user through OAuth setup."""
    print("\n=== Google Docs BFS Exporter - OAuth Setup ===\n")
    print("To use this tool, you need to set up OAuth credentials:")
    print("\n1. Go to: https://console.cloud.google.com/")
    print("2. Create a new project (or select existing)")
    print("3. Enable APIs:")
    print("   - Google Docs API")
    print("   - Google Drive API")
    print("4. Go to 'Credentials' → 'Create Credentials' → 'OAuth client ID'")
    print("5. Choose 'Desktop app' as application type")
    print("6. Download the JSON file and save it as 'credentials.json' in this directory")
    print("\n7. Run this script again without --setup flag\n")
    print("Note: If you've previously authenticated and want to use --drive export,")
    print("you may need to delete token.pickle to re-authenticate with new scopes.\n")
    
    if os.path.exists(CREDENTIALS_PATH):
        print(f"✓ Found {CREDENTIALS_PATH}")
        print("You can now run the crawler!")
    else:
        print(f"⚠ {CREDENTIALS_PATH} not found. Please complete the steps above.")


def main():
    parser = argparse.ArgumentParser(
        description='Export Google Docs by following links (BFS)',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  # First time setup
  python main.py --setup
  
  # Export as Google Docs copies (default, local .docx files)
  python main.py --seed-id 1abc_YOUR_DOC_ID_xyz
  
  # Export as markdown files (local)
  python main.py --seed-id 1abc_YOUR_DOC_ID_xyz --format md
  
  # Save copies to Google Drive folder (preserves as Google Docs)
  python main.py --seed-id 1abc_YOUR_DOC_ID_xyz --drive 1xyz_FOLDER_ID_abc
  
  # Save markdown to Drive with localized links (links point to other Drive files)
  python main.py --seed-id 1abc_YOUR_DOC_ID_xyz --drive 1xyz_FOLDER_ID_abc --format md --localize-links
  
  # Automatically request access to documents you don't have permission for
  python main.py --seed-id 1abc_YOUR_DOC_ID_xyz --request-access
  
  # Limit to 50 documents
  python main.py --seed-id 1abc_YOUR_DOC_ID_xyz --max-docs 50
        """
    )
    
    parser.add_argument('--setup', action='store_true',
                        help='Show OAuth setup instructions')
    parser.add_argument('--seed-id', type=str,
                        help='Google Doc ID to start crawling from')
    parser.add_argument('--format', type=str, default='docx', choices=['md', 'docx'],
                        help='Export format: md (markdown) or docx (default: docx)')
    parser.add_argument('--drive', type=str, metavar='FOLDER_ID',
                        help='Save copies to Google Drive folder instead of local export (provide folder ID)')
    parser.add_argument('--localize-links', action='store_true',
                        help='Convert Google Docs links to point to exported documents (local files or Drive URLs)')
    parser.add_argument('--request-access', action='store_true',
                        help='Automatically request access when encountering permission errors')
    parser.add_argument('--max-docs', type=int, default=100,
                        help='Maximum number of documents to export (default: 100)')
    
    args = parser.parse_args()
    
    if args.setup:
        setup_oauth()
        return
    
    if not args.seed_id:
        parser.error("--seed-id is required (or use --setup for first-time configuration)")
    
    # Validate arguments
    # Note: --localize-links works with --drive for markdown (updates links to Drive URLs)
    # For docx format with Drive, links cannot be easily updated (would require Docs API modifications)
    
    # Run the crawler
    crawler = GoogleDocsCrawler(
        export_format=args.format,
        localize_links=args.localize_links,
        request_access=args.request_access,
        drive_folder_id=args.drive
    )
    crawler.crawl_and_export(args.seed_id, max_docs=args.max_docs)


if __name__ == "__main__":
    main()