url_threat_analyzer.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
URL Threat Analyzer — SIEM Integration Tool
Multi-source threat intel: VirusTotal, URLScan.io

Analyzes bulk URLs for malicious indicators using multiple threat
intelligence sources. Designed for SIEM pipeline integration.

Usage:
    python url_threat_analyzer.py -u "https://example.com"
    python url_threat_analyzer.py -f urls.txt
    python url_threat_analyzer.py -f urls.txt -o results.json --format json
    python url_threat_analyzer.py -f urls.txt -o results.csv --format csv
    echo "https://example.com" | python url_threat_analyzer.py --stdin

Environment Variables (set these in config.env):
    VT_API_KEY       — VirusTotal API key  (free tier: 4 req/min)
    URLSCAN_API_KEY  — URLScan.io API key  (free tier: 60 req/min)
"""

import argparse
import base64
import csv
import io
import json
import logging
import os
import re
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse

try:
    import requests
    from requests.adapters import HTTPAdapter
    from urllib3.util.retry import Retry
except ImportError:
    print("[INFO] 'requests' library not found — auto-installing...")
    import subprocess as _sp
    _r = _sp.run([sys.executable, "-m", "pip", "install", "requests"], check=False)
    if _r.returncode != 0:
        print("[ERROR] Auto-install failed. Run manually:")
        print(f"  {sys.executable} -m pip install requests")
        sys.exit(1)
    import requests
    from requests.adapters import HTTPAdapter
    from urllib3.util.retry import Retry
    print("[INFO] 'requests' installed. Continuing...")

# Ensure UTF-8 output on Windows (avoids CP1252 emoji encoding errors)
if sys.stdout.encoding and sys.stdout.encoding.lower() not in ("utf-8", "utf_8"):
    import io
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
if sys.stderr.encoding and sys.stderr.encoding.lower() not in ("utf-8", "utf_8"):
    import io
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")


# ---------------------------------------------------------------------------
# CONFIGURATION
# ---------------------------------------------------------------------------

def _load_env_file(env_path: str = "config.env") -> None:
    """Simple .env file parser — no external dependency needed."""
    if Path(env_path).exists():
        with open(env_path, "r") as f:
            for line in f:
                line = line.strip()
                if line and not line.startswith("#") and "=" in line:
                    key, _, value = line.partition("=")
                    os.environ.setdefault(
                        key.strip(), value.strip().strip('"').strip("'")
                    )

_load_env_file()

VT_API_KEY      = os.getenv("VT_API_KEY", "")
URLSCAN_API_KEY = os.getenv("URLSCAN_API_KEY", "")

VT_BASE       = "https://www.virustotal.com/api/v3"
URLSCAN_BASE  = "https://urlscan.io/api/v1"

VT_RATE_LIMIT = 4    # free tier: 4 requests/minute
MALICIOUS_THRESHOLD  = 3
SUSPICIOUS_THRESHOLD = 1


# ---------------------------------------------------------------------------
# LOGGING
# ---------------------------------------------------------------------------

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
    handlers=[logging.StreamHandler(sys.stderr)],
)
log = logging.getLogger("url_threat_analyzer")


# ---------------------------------------------------------------------------
# DATA MODELS
# ---------------------------------------------------------------------------

@dataclass
class VTResult:
    malicious:         int = 0
    suspicious:        int = 0
    harmless:          int = 0
    undetected:        int = 0
    total_engines:     int = 0
    categories:        list = field(default_factory=list)
    last_analysis_date: str = ""
    reputation:        int = 0
    error:             str = ""


@dataclass
class URLScanResult:
    verdict:        str = ""
    score:          int = 0
    tags:           list = field(default_factory=list)
    report_url:     str = ""
    screenshot_url: str = ""
    country:        str = ""
    server:         str = ""
    error:          str = ""


@dataclass
class ThreatResult:
    url:               str = ""
    domain:            str = ""
    timestamp:         str = ""
    verdict:           str = "UNKNOWN"   # MALICIOUS | SUSPICIOUS | CLEAN | UNKNOWN
    confidence:        str = ""          # HIGH | MEDIUM | LOW
    risk_score:        int = 0           # 0–100 composite
    virustotal:        VTResult = field(default_factory=VTResult)
    urlscan:           URLScanResult = field(default_factory=URLScanResult)
    indicators:        list = field(default_factory=list)
    threat_categories: list = field(default_factory=list)
    alert_id:          str = ""
    source:            str = "url_threat_analyzer"


# ---------------------------------------------------------------------------
# HTTP SESSION
# ---------------------------------------------------------------------------

def _make_session(retries: int = 3) -> requests.Session:
    session = requests.Session()
    retry = Retry(
        total=retries,
        backoff_factor=1.5,
        status_forcelist=[429, 500, 502, 503, 504],
        allowed_methods=["GET", "POST"],
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("https://", adapter)
    session.mount("http://", adapter)
    session.headers.update({"User-Agent": "SIEM-URLAnalyzer/1.0"})
    return session


# ---------------------------------------------------------------------------
# VIRUSTOTAL
# ---------------------------------------------------------------------------

class VirusTotalChecker:
    """Queries VirusTotal v3 API for URL reputation."""

    def __init__(self, api_key: str):
        self.api_key = api_key
        self.session = _make_session()
        self.session.headers.update({"x-apikey": api_key})
        self._last_call    = 0.0
        self._min_interval = 60.0 / VT_RATE_LIMIT

    def _rate_limit(self) -> None:
        elapsed = time.time() - self._last_call
        if elapsed < self._min_interval:
            time.sleep(self._min_interval - elapsed)
        self._last_call = time.time()

    @staticmethod
    def _url_id(url: str) -> str:
        return base64.urlsafe_b64encode(url.encode()).decode().rstrip("=")

    def check(self, url: str) -> VTResult:
        if not self.api_key:
            return VTResult(error="VT_API_KEY not configured")

        self._rate_limit()
        url_id   = self._url_id(url)
        endpoint = f"{VT_BASE}/urls/{url_id}"

        try:
            resp = self.session.get(endpoint, timeout=15)

            # URL not yet analysed — submit it
            if resp.status_code == 404:
                log.info("  [VT] URL not cached, submitting for analysis...")
                sub = self.session.post(
                    f"{VT_BASE}/urls", data={"url": url}, timeout=15
                )
                if sub.status_code not in (200, 201):
                    return VTResult(error=f"VT submit failed: {sub.status_code}")
                for _ in range(6):
                    time.sleep(5)
                    resp = self.session.get(endpoint, timeout=15)
                    if resp.status_code == 200:
                        break
                else:
                    return VTResult(error="VT analysis timed out (30s)")

            if resp.status_code == 429:
                return VTResult(error="VT rate limit exceeded — slow down or upgrade plan")
            if resp.status_code != 200:
                return VTResult(error=f"VT HTTP {resp.status_code}")

            attr  = resp.json().get("data", {}).get("attributes", {})
            stats = attr.get("last_analysis_stats", {})
            cats  = list(set(attr.get("categories", {}).values()))
            ts    = attr.get("last_analysis_date", 0)

            return VTResult(
                malicious          = stats.get("malicious", 0),
                suspicious         = stats.get("suspicious", 0),
                harmless           = stats.get("harmless", 0),
                undetected         = stats.get("undetected", 0),
                total_engines      = sum(stats.values()),
                categories         = cats,
                last_analysis_date = (
                    datetime.fromtimestamp(ts, tz=timezone.utc).isoformat() if ts else ""
                ),
                reputation         = attr.get("reputation", 0),
            )

        except requests.exceptions.Timeout:
            return VTResult(error="VT request timed out")
        except requests.exceptions.RequestException as e:
            return VTResult(error=f"VT request error: {e}")
        except Exception as e:
            return VTResult(error=f"VT unexpected error: {e}")


# ---------------------------------------------------------------------------
# URLSCAN.IO
# ---------------------------------------------------------------------------

class URLScanChecker:
    """Queries URLScan.io — searches for cached scans, submits new if none found."""

    def __init__(self, api_key: str):
        self.api_key = api_key
        self.session = _make_session()
        if api_key:
            self.session.headers.update({"API-Key": api_key})

    def check(self, url: str) -> URLScanResult:
        if not self.api_key:
            return URLScanResult(error="URLSCAN_API_KEY not configured")

        try:
            # 1. Search for existing scan
            search = self.session.get(
                f"{URLSCAN_BASE}/search/",
                params={"q": f'page.url:"{url}"', "size": 1},
                timeout=15,
            )
            scan_uuid = None
            if search.status_code == 200:
                hits = search.json().get("results", [])
                if hits:
                    scan_uuid = hits[0].get("task", {}).get("uuid")
                    log.info(f"  [URLScan] Using cached scan: {scan_uuid}")

            # 2. Submit new scan if nothing found
            if not scan_uuid:
                log.info("  [URLScan] Submitting new scan...")
                sub = self.session.post(
                    f"{URLSCAN_BASE}/scan/",
                    json={"url": url, "visibility": "public"},
                    timeout=15,
                )
                if sub.status_code not in (200, 201):
                    return URLScanResult(error=f"URLScan submit failed: {sub.status_code}")
                scan_uuid = sub.json().get("uuid")
                if not scan_uuid:
                    return URLScanResult(error="URLScan: no UUID returned")

                log.info(f"  [URLScan] Waiting for scan {scan_uuid} (up to 60s)...")
                result_resp = None
                for _ in range(12):
                    time.sleep(5)
                    r = self.session.get(
                        f"{URLSCAN_BASE}/result/{scan_uuid}/", timeout=15
                    )
                    if r.status_code == 200:
                        result_resp = r
                        break
                if result_resp is None:
                    return URLScanResult(error="URLScan: result not ready within 60s")
            else:
                result_resp = self.session.get(
                    f"{URLSCAN_BASE}/result/{scan_uuid}/", timeout=15
                )

            if result_resp.status_code != 200:
                return URLScanResult(error=f"URLScan result HTTP {result_resp.status_code}")

            data     = result_resp.json()
            verdicts = data.get("verdicts", {}).get("overall", {})
            page     = data.get("page", {})

            return URLScanResult(
                verdict        = (
                    "malicious"  if verdicts.get("malicious")  else
                    "suspicious" if verdicts.get("suspicious") else "benign"
                ),
                score          = verdicts.get("score", 0),
                tags           = verdicts.get("tags", []),
                report_url     = f"https://urlscan.io/result/{scan_uuid}/",
                screenshot_url = f"https://urlscan.io/screenshots/{scan_uuid}.png",
                country        = page.get("country", ""),
                server         = page.get("server", ""),
            )

        except requests.exceptions.Timeout:
            return URLScanResult(error="URLScan request timed out")
        except requests.exceptions.RequestException as e:
            return URLScanResult(error=f"URLScan request error: {e}")
        except Exception as e:
            return URLScanResult(error=f"URLScan unexpected error: {e}")


# ---------------------------------------------------------------------------
# STATIC / HEURISTIC ANALYSIS  (no API key required)
# ---------------------------------------------------------------------------

SUSPICIOUS_TLDS = {
    ".tk", ".ml", ".ga", ".cf", ".gq",
    ".xyz", ".top", ".club", ".work", ".online",
    ".icu", ".cyou", ".buzz", ".fun", ".vip",
}

PHISHING_KEYWORDS = [
    "login", "signin", "verify", "account", "secure", "update",
    "confirm", "banking", "paypal", "amazon", "microsoft", "apple",
    "google", "facebook", "netflix", "support", "helpdesk",
    "password", "credential", "wallet", "crypto", "urgent", "free",
]

URL_SHORTENERS = {
    "bit.ly", "tinyurl.com", "t.co", "goo.gl", "ow.ly",
    "short.link", "rebrand.ly", "is.gd", "cutt.ly", "tiny.cc",
    "lnkd.in", "buff.ly", "adf.ly", "linktr.ee",
}

KNOWN_BRANDS = [
    "paypal", "amazon", "microsoft", "apple", "google", "facebook",
    "netflix", "instagram", "twitter", "chase", "wellsfargo",
    "bankofamerica", "dropbox", "linkedin", "docusign",
]

MALWARE_EXTENSIONS = [
    ".exe", ".bat", ".ps1", ".vbs", ".js", ".jar", ".cmd",
    ".scr", ".msi", ".hta", ".pif", ".reg",
]


def static_analysis(url: str) -> tuple[list[str], int]:
    """
    Heuristic analysis of URL structure.
    Returns (indicators, risk_score_delta 0-100).
    """
    indicators: list[str] = []
    score = 0

    try:
        parsed = urlparse(url)
        domain = parsed.netloc.lower().lstrip("www.")
        path   = parsed.path.lower()
        query  = parsed.query.lower()
        full   = path + "?" + query
    except Exception:
        return ["Failed to parse URL structure"], 20

    # 1. No HTTPS
    if parsed.scheme == "http":
        indicators.append("Plain HTTP used (no TLS encryption)")
        score += 10

    # 2. Raw IP address as host
    if re.match(r"^\d{1,3}(\.\d{1,3}){3}(:\d+)?$", domain):
        indicators.append("Raw IP address used instead of domain name")
        score += 25

    # 3. Suspicious free TLD
    for tld in SUSPICIOUS_TLDS:
        if domain.endswith(tld):
            indicators.append(f"Suspicious TLD: {tld}")
            score += 15
            break

    # 4. Phishing keywords
    found = [kw for kw in PHISHING_KEYWORDS if kw in full or kw in domain]
    if found:
        indicators.append(f"Phishing keywords: {', '.join(found[:5])}")
        score += min(len(found) * 8, 30)

    # 5. URL shortener
    if domain in URL_SHORTENERS:
        indicators.append(f"URL shortener ({domain}) hides real destination")
        score += 20

    # 6. Excessive subdomains
    parts = domain.split(".")
    if len(parts) > 4:
        indicators.append(f"Excessive subdomains ({len(parts)} levels) — possible lookalike")
        score += 20

    # 7. Abnormally long URL
    if len(url) > 200:
        indicators.append(f"Unusually long URL ({len(url)} chars) — possible obfuscation")
        score += 10

    # 8. Heavy URL encoding
    if url.count("%") > 5:
        indicators.append("Heavy URL encoding — possible character obfuscation")
        score += 15

    # 9. Malware file extension in path
    for ext in MALWARE_EXTENSIONS:
        if ext in path:
            indicators.append(f"Suspicious file extension in path: {ext}")
            score += 30
            break

    # 10. Brand impersonation (brand in subdomain, not root)
    root = ".".join(parts[-2:]) if len(parts) >= 2 else domain
    for brand in KNOWN_BRANDS:
        if brand in domain and brand not in root:
            indicators.append(f"Brand impersonation: '{brand}' in subdomain of unrelated root domain")
            score += 35
            break

    # 11. Multiple redirects signalled by path
    if url.count("//") > 1:
        indicators.append("Multiple '//' in URL — possible open redirect")
        score += 15

    return indicators, min(score, 100)


# ---------------------------------------------------------------------------
# VERDICT ENGINE
# ---------------------------------------------------------------------------

def compute_verdict(result: ThreatResult) -> ThreatResult:
    score = result.risk_score  # seed with heuristic score
    vt, us = result.virustotal, result.urlscan

    if not vt.error:
        if vt.malicious >= MALICIOUS_THRESHOLD:
            score += 50
        elif vt.malicious >= SUSPICIOUS_THRESHOLD:
            score += 25
        if vt.suspicious >= SUSPICIOUS_THRESHOLD:
            score += 10
        if vt.reputation < -10:
            score += 15
        result.threat_categories.extend(vt.categories)

    if not us.error:
        if us.verdict == "malicious":
            score += 40
        elif us.verdict == "suspicious":
            score += 20
        score += min(us.score // 5, 20)
        result.threat_categories.extend(us.tags)

    result.threat_categories = list(set(result.threat_categories))
    result.risk_score = min(score, 100)

    if result.risk_score >= 60:
        result.verdict    = "MALICIOUS"
        result.confidence = "HIGH" if result.risk_score >= 80 else "MEDIUM"
    elif result.risk_score >= 30:
        result.verdict    = "SUSPICIOUS"
        result.confidence = "MEDIUM" if result.risk_score >= 45 else "LOW"
    else:
        result.verdict    = "CLEAN"
        result.confidence = "HIGH" if result.risk_score <= 10 else "MEDIUM"

    return result


# ---------------------------------------------------------------------------
# MAIN ANALYZER
# ---------------------------------------------------------------------------

class URLThreatAnalyzer:

    def __init__(self, use_vt: bool = True, use_urlscan: bool = True):
        self.vt = VirusTotalChecker(VT_API_KEY)   if (use_vt      and VT_API_KEY)       else None
        self.us = URLScanChecker(URLSCAN_API_KEY)  if (use_urlscan and URLSCAN_API_KEY)  else None

        if not self.vt:
            log.warning("VirusTotal disabled — set VT_API_KEY in config.env")
        if not self.us:
            log.warning("URLScan.io disabled — set URLSCAN_API_KEY in config.env")

    def analyze_url(self, url: str, alert_id: str = "") -> ThreatResult:
        url = url.strip()
        if not url.startswith(("http://", "https://")):
            url = "https://" + url

        log.info(f"[+] Analyzing: {url[:80]}")
        parsed = urlparse(url)

        result = ThreatResult(
            url       = url,
            domain    = parsed.netloc,
            timestamp = datetime.now(tz=timezone.utc).isoformat(),
            alert_id  = alert_id,
        )

        # 1. Static heuristics — always run (no API required)
        log.info("  [Heuristic] Static analysis...")
        indicators, heuristic_score = static_analysis(url)
        result.indicators  = indicators
        result.risk_score  = heuristic_score

        # 2. VirusTotal
        if self.vt:
            log.info("  [VirusTotal] Querying reputation...")
            result.virustotal = self.vt.check(url)
            if result.virustotal.error:
                log.warning(f"  [VirusTotal] {result.virustotal.error}")
            else:
                log.info(
                    f"  [VirusTotal] {result.virustotal.malicious} malicious / "
                    f"{result.virustotal.total_engines} engines"
                )

        # 3. URLScan.io
        if self.us:
            log.info("  [URLScan] Querying...")
            result.urlscan = self.us.check(url)
            if result.urlscan.error:
                log.warning(f"  [URLScan] {result.urlscan.error}")
            else:
                log.info(
                    f"  [URLScan] verdict={result.urlscan.verdict} score={result.urlscan.score}"
                )

        # 4. Compute composite verdict
        result = compute_verdict(result)

        icon = {"MALICIOUS": "[!]", "SUSPICIOUS": "[?]", "CLEAN": "[+]"}.get(result.verdict, "[-]")
        log.info(f"  {icon} Result: {result.verdict} | Risk: {result.risk_score}/100")

        return result

    def analyze_bulk(
        self, urls: list[str], max_workers: int = 1, alert_id: str = ""
    ) -> list[ThreatResult]:
        total   = len(urls)
        results = []

        log.info(f"\n{'='*60}")
        log.info(f"  URL THREAT ANALYZER — {total} URL(s) queued")
        log.info(f"{'='*60}\n")

        if max_workers <= 1:
            for i, url in enumerate(urls, 1):
                log.info(f"--- [{i}/{total}] ---")
                results.append(self.analyze_url(url, alert_id=alert_id))
        else:
            with ThreadPoolExecutor(max_workers=max_workers) as pool:
                futures = {pool.submit(self.analyze_url, u, alert_id): u for u in urls}
                done = 0
                for future in as_completed(futures):
                    done += 1
                    log.info(f"--- [{done}/{total}] completed ---")
                    results.append(future.result())

        return results


# ---------------------------------------------------------------------------
# OUTPUT FORMATTERS
# ---------------------------------------------------------------------------

def format_json(results: list[ThreatResult], pretty: bool = True) -> str:
    return json.dumps([asdict(r) for r in results], indent=2 if pretty else None)


def format_csv(results: list[ThreatResult]) -> str:
    output = io.StringIO()
    fields = [
        "timestamp", "url", "domain", "verdict", "confidence", "risk_score",
        "vt_malicious", "vt_suspicious", "vt_harmless", "vt_total_engines",
        "vt_reputation", "vt_categories", "vt_last_analysis",
        "urlscan_verdict", "urlscan_score", "urlscan_country",
        "urlscan_server", "urlscan_report_url",
        "indicators", "threat_categories", "alert_id",
    ]
    writer = csv.DictWriter(output, fieldnames=fields, extrasaction="ignore")
    writer.writeheader()
    for r in results:
        writer.writerow({
            "timestamp"          : r.timestamp,
            "url"                : r.url,
            "domain"             : r.domain,
            "verdict"            : r.verdict,
            "confidence"         : r.confidence,
            "risk_score"         : r.risk_score,
            "vt_malicious"       : r.virustotal.malicious,
            "vt_suspicious"      : r.virustotal.suspicious,
            "vt_harmless"        : r.virustotal.harmless,
            "vt_total_engines"   : r.virustotal.total_engines,
            "vt_reputation"      : r.virustotal.reputation,
            "vt_categories"      : ", ".join(r.virustotal.categories),
            "vt_last_analysis"   : r.virustotal.last_analysis_date,
            "urlscan_verdict"    : r.urlscan.verdict,
            "urlscan_score"      : r.urlscan.score,
            "urlscan_country"    : r.urlscan.country,
            "urlscan_server"     : r.urlscan.server,
            "urlscan_report_url" : r.urlscan.report_url,
            "indicators"         : " | ".join(r.indicators),
            "threat_categories"  : ", ".join(r.threat_categories),
            "alert_id"           : r.alert_id,
        })
    return output.getvalue()


def print_console_report(results: list[ThreatResult]) -> None:
    malicious  = [r for r in results if r.verdict == "MALICIOUS"]
    suspicious = [r for r in results if r.verdict == "SUSPICIOUS"]
    clean      = [r for r in results if r.verdict == "CLEAN"]

    sep = "═" * 70

    print(f"\n{sep}")
    print("  URL THREAT ANALYSIS REPORT")
    print(sep)
    print(f"  Analyzed     : {len(results)} URL(s)")
    print(f"  [!] Malicious  : {len(malicious)}")
    print(f"  [?] Suspicious : {len(suspicious)}")
    print(f"  [+] Clean      : {len(clean)}")
    print(sep)

    for r in results:
        icon = {"MALICIOUS": "[!]", "SUSPICIOUS": "[?]", "CLEAN": "[+]"}.get(r.verdict, "[-]")
        print(f"\n{icon} [{r.verdict}] {r.url}")
        print(f"   Risk Score   : {r.risk_score}/100  |  Confidence: {r.confidence}")

        vt = r.virustotal
        if not vt.error and vt.total_engines:
            print(
                f"   VirusTotal   : {vt.malicious} malicious / "
                f"{vt.suspicious} suspicious / {vt.total_engines} engines"
            )
            if vt.reputation != 0:
                print(f"   VT Reputation: {vt.reputation}")
        elif vt.error:
            print(f"   VirusTotal   : [WARN] {vt.error}")

        us = r.urlscan
        if not us.error and us.verdict:
            print(f"   URLScan.io   : {us.verdict.upper()} (score: {us.score}/100)")
            if us.country:
                print(f"   Server       : {us.server or 'N/A'}  |  Country: {us.country}")
            if us.report_url:
                print(f"   Full Report  : {us.report_url}")
        elif us.error:
            print(f"   URLScan.io   : [WARN] {us.error}")

        if r.indicators:
            print("   Indicators   :")
            for ind in r.indicators:
                print(f"      * {ind}")

        if r.threat_categories:
            print(f"   Categories   : {', '.join(r.threat_categories)}")

    print(f"\n{sep}")

    if malicious or suspicious:
        print("\n  [WARN] ACTIONABLE ITEMS — Investigate immediately:")
        for r in malicious + suspicious:
            print(f"     [{r.verdict:<10}] Risk {r.risk_score:>3}/100  |  {r.url}")
    print()


# ---------------------------------------------------------------------------
# UTILITIES
# ---------------------------------------------------------------------------

def read_urls_from_file(path: str) -> list[str]:
    with open(path, "r", encoding="utf-8") as f:
        return [
            line.strip() for line in f
            if line.strip() and not line.strip().startswith("#")
        ]


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        description="Bulk URL Threat Analyzer — SIEM Integration Tool",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python url_threat_analyzer.py -u "https://suspicious-site.com/login"
  python url_threat_analyzer.py -f urls.txt
  python url_threat_analyzer.py -f urls.txt -o results.json --format json
  python url_threat_analyzer.py -f urls.txt -o results.csv --format csv
  echo "https://bad-url.com" | python url_threat_analyzer.py --stdin
  python url_threat_analyzer.py -f urls.txt --alert-id "ALERT-2024-001"
        """,
    )

    grp = parser.add_mutually_exclusive_group(required=True)
    grp.add_argument("-u", "--url",   help="Single URL to analyze")
    grp.add_argument("-f", "--file",  help="File with URLs (one per line, # = comment)")
    grp.add_argument("--stdin",       action="store_true", help="Read URLs from stdin")

    parser.add_argument("-o", "--output",   help="Output file path (default: stdout)")
    parser.add_argument(
        "--format", choices=["json", "csv", "console"], default="console",
        help="Output format (default: console)"
    )
    parser.add_argument("--alert-id",    default="", help="SIEM alert ID to tag results")
    parser.add_argument("--no-vt",       action="store_true", help="Skip VirusTotal")
    parser.add_argument("--no-urlscan",  action="store_true", help="Skip URLScan.io")
    parser.add_argument(
        "--workers", type=int, default=1,
        help="Parallel workers (default 1; only increase with paid API keys)"
    )
    parser.add_argument("--verbose",     action="store_true", help="Debug logging")

    args = parser.parse_args()

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    # Collect URLs
    if args.url:
        urls = [args.url]
    elif args.file:
        if not Path(args.file).exists():
            log.error(f"File not found: {args.file}")
            sys.exit(1)
        urls = read_urls_from_file(args.file)
    else:
        urls = [
            line.strip() for line in sys.stdin
            if line.strip() and not line.strip().startswith("#")
        ]

    if not urls:
        log.error("No URLs provided.")
        sys.exit(1)

    log.info(f"Loaded {len(urls)} URL(s)")

    analyzer = URLThreatAnalyzer(
        use_vt      = not args.no_vt,
        use_urlscan = not args.no_urlscan,
    )

    results = analyzer.analyze_bulk(urls, max_workers=args.workers, alert_id=args.alert_id)

    if args.format == "json":
        output_str = format_json(results)
    elif args.format == "csv":
        output_str = format_csv(results)
    else:
        print_console_report(results)
        output_str = None

    if output_str is not None:
        if args.output:
            with open(args.output, "w", encoding="utf-8") as f:
                f.write(output_str)
            log.info(f"Saved to: {args.output}")
        else:
            print(output_str)

    # Also print summary when saving to file
    if args.output and args.format != "console":
        print_console_report(results)

    # Exit codes useful for SIEM orchestrators:
    #   0 = all clean   1 = suspicious found   2 = malicious found
    if any(r.verdict == "MALICIOUS"  for r in results):
        sys.exit(2)
    elif any(r.verdict == "SUSPICIOUS" for r in results):
        sys.exit(1)
    sys.exit(0)


if __name__ == "__main__":
    main()