cli_scraper.py

import asyncio
import os
import sys
from dotenv import load_dotenv

# Add current directory to path so imports work
sys.path.append(os.getcwd())

# Load environment variables
load_dotenv()

from backend.app.services.legal_scraper import legal_scraper

async def main():
    print("=== LegalAI Scraper Tool (Firecrawl Enabled) ===")
    
    # Check for Firecrawl Key
    api_key = os.getenv("FIRECRAWL_API_KEY")
    if api_key:
        print(f"✅ Firecrawl API Key found: {api_key[:5]}... (Will use Firecrawl)")
    else:
        print("⚠️  Firecrawl API Key NOT found. (Will fallback to basic scraper)")
        print("   To enable Firecrawl, add FIRECRAWL_API_KEY=fc-xxx to your .env file")

    while True:
        print("\n------------------------------------------------")
        url = input("Enter URL to scrape (or 'q' to quit): ").strip()
        
        if url.lower() in ('q', 'quit', 'exit'):
            break
            
        if not url:
            continue

        print(f"\nProcessing: {url}...")
        try:
            # Call the scraper
            result = await legal_scraper.scrape_and_ingest(url, category="Manual Input")
            
            if result.get("status") == "success":
                print("\n✅ Success!")
                print(f"Law ID: {result.get('law_id')}")
                print("Data has been saved to the database.")
            else:
                print("\n❌ Error:")
                print(result.get("message"))
                
        except Exception as e:
            print(f"\n❌ Unexpected Error: {e}")

if __name__ == "__main__":
    try:
        asyncio.run(main())
    except KeyboardInterrupt:
        print("\nExiting...")