app.py

#!/usr/bin/env python3
"""
Dorker Pro OSINT Edition v5.0 - Interface Streamlit
Multi-Domaines avec thèmes visuels interchangeables
+ Mode Chasseur de fuites
+ Extraction IOC
+ Améliorations UX/UI
"""

import streamlit as st
import asyncio
import aiohttp
import aiosqlite
import json
import time
import random
import urllib.parse
import hashlib
import csv
import io
import re
import plotly.express as px
import plotly.graph_objects as go
from pathlib import Path
from typing import List, Dict, Set, Optional, Tuple
from dataclasses import dataclass, asdict, field
from datetime import datetime
from collections import defaultdict, Counter
from email_validator import validate_email, EmailNotValidError
import phonenumbers
from phonenumbers import carrier, geocoder, timezone
import logging
import pandas as pd

# ============================================================================
# THÈMES CSS (version améliorée UX)
# ============================================================================
THEMES = {
    "Terminal Rétro": {
        "icon": "🖥️",
        "css": """
        @import url('https://fonts.googleapis.com/css2?family=VT323&family=Share+Tech+Mono&display=swap');

        :root {
            --bg-primary: #0a0a0a;
            --bg-secondary: #111111;
            --bg-card: #0d1a0d;
            --accent: #00ff41;
            --accent-dim: #00aa2a;
            --accent-glow: rgba(0,255,65,0.15);
            --text-primary: #00ff41;
            --text-secondary: #00aa2a;
            --text-dim: #005510;
            --border: #00ff41;
            --danger: #ff0040;
            --warning: #ffaa00;
            --info: #00aaff;
            --font-main: 'Share Tech Mono', monospace;
            --font-display: 'VT323', monospace;
        }

        .stApp {
            background-color: var(--bg-primary) !important;
            background-image:
                repeating-linear-gradient(
                    0deg,
                    transparent,
                    transparent 2px,
                    rgba(0, 255, 65, 0.02) 2px,
                    rgba(0, 255, 65, 0.02) 4px
                ) !important;
            font-family: var(--font-main) !important;
            color: var(--text-primary) !important;
            transition: all 0.3s ease !important;
        }

        .stApp::before {
            content: '';
            position: fixed;
            top: 0; left: 0; right: 0; bottom: 0;
            background: repeating-linear-gradient(
                0deg,
                rgba(0,0,0,0.15) 0px,
                rgba(0,0,0,0.15) 1px,
                transparent 1px,
                transparent 2px
            );
            pointer-events: none;
            z-index: 9999;
        }

        h1, h2, h3, h4 {
            font-family: var(--font-display) !important;
            color: var(--accent) !important;
            text-shadow: 0 0 10px var(--accent), 0 0 20px var(--accent-dim) !important;
            letter-spacing: 3px !important;
        }

        h1 { font-size: 3rem !important; }

        .stSidebar {
            background-color: #050f05 !important;
            border-right: 1px solid var(--accent-dim) !important;
            transition: transform 0.3s ease !important;
        }

        .stButton > button {
            background: transparent !important;
            color: var(--accent) !important;
            border: 1px solid var(--accent) !important;
            border-radius: 0 !important;
            font-family: var(--font-main) !important;
            font-size: 0.9rem !important;
            letter-spacing: 2px !important;
            text-transform: uppercase !important;
            box-shadow: 0 0 10px var(--accent-glow) !important;
            transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
        }

        .stButton > button:hover {
            background: var(--accent-glow) !important;
            box-shadow: 0 0 20px var(--accent), 0 0 40px var(--accent-dim) !important;
            transform: scale(1.02) !important;
        }

        /* Animation de chargement cyber */
        @keyframes pulse-glow {
            0% { box-shadow: 0 0 5px var(--accent); }
            100% { box-shadow: 0 0 20px var(--accent), 0 0 30px var(--accent-dim); }
        }

        div[data-testid="stMetric"] {
            background: var(--bg-card) !important;
            border: 1px solid var(--accent-dim) !important;
            border-radius: 0 !important;
            padding: 10px !important;
            transition: all 0.3s ease !important;
        }

        div[data-testid="stMetric"]:hover {
            transform: translateY(-2px) !important;
            border-color: var(--accent) !important;
            box-shadow: 0 0 15px var(--accent-glow) !important;
        }

        .score-high {
            color: #00ff41;
            font-weight: bold;
            animation: pulse-glow 2s infinite !important;
        }
        .score-med { color: #ffaa00; }
        .score-low { color: #ff0040; }

        .ioc-email {
            background: rgba(0,255,65,0.1);
            border-left: 3px solid #00aaff;
            padding: 8px;
            margin: 4px 0;
            font-family: monospace;
        }
        .ioc-ip {
            background: rgba(255,170,0,0.1);
            border-left: 3px solid #ffaa00;
            padding: 8px;
            margin: 4px 0;
        }
        .leak-alert {
            background: rgba(255,0,64,0.2);
            border: 2px solid #ff0040;
            animation: pulse-glow 1s infinite;
            padding: 12px;
            margin: 8px 0;
        }

        .compact-mode .stDataFrame {
            font-size: 0.8rem !important;
        }

        .stProgress > div > div > div {
            background: linear-gradient(90deg, var(--accent-dim), var(--accent), var(--accent-dim)) !important;
            background-size: 200% 100% !important;
            animation: shimmer 1s infinite linear !important;
        }

        @keyframes shimmer {
            0% { background-position: 200% 0; }
            100% { background-position: -200% 0; }
        }

        .terminal-box {
            background: #000;
            border: 1px solid #00ff41;
            border-radius: 0;
            padding: 16px;
            font-family: 'Share Tech Mono', monospace;
            font-size: 0.85rem;
            color: #00ff41;
            box-shadow: 0 0 20px rgba(0,255,65,0.1), inset 0 0 20px rgba(0,0,0,0.5);
            white-space: pre-wrap;
            max-height: 400px;
            overflow-y: auto;
        }
        """
    },
    "CyberPunk 2077": {
        "icon": "⚡",
        "css": """
        @import url('https://fonts.googleapis.com/css2?family=Orbitron:wght@400;700;900&family=Rajdhani:wght@300;500;700&display=swap');

        :root {
            --bg-primary: #0a0014;
            --bg-secondary: #120020;
            --bg-card: #1a0030;
            --accent: #fcee09;
            --accent2: #ff003c;
            --accent3: #00f0ff;
            --accent-glow: rgba(252,238,9,0.15);
            --text-primary: #f0e6ff;
            --text-secondary: #9b7fd4;
            --border: #fcee09;
            --font-main: 'Rajdhani', sans-serif;
            --font-display: 'Orbitron', monospace;
        }

        .stApp {
            background-color: var(--bg-primary) !important;
            background-image:
                radial-gradient(ellipse at 20% 50%, rgba(100,0,255,0.08) 0%, transparent 50%),
                radial-gradient(ellipse at 80% 20%, rgba(255,0,60,0.06) 0%, transparent 50%),
                radial-gradient(ellipse at 60% 80%, rgba(0,240,255,0.06) 0%, transparent 50%) !important;
            font-family: var(--font-main) !important;
            color: var(--text-primary) !important;
        }

        h1, h2, h3, h4 {
            font-family: var(--font-display) !important;
            color: var(--accent) !important;
            text-transform: uppercase !important;
            letter-spacing: 4px !important;
        }

        h1 {
            font-size: 2.5rem !important;
            text-shadow: 0 0 10px var(--accent), 0 0 30px rgba(252,238,9,0.5), -2px 2px 0 var(--accent2) !important;
        }

        h2 {
            text-shadow: 0 0 8px var(--accent3) !important;
            color: var(--accent3) !important;
        }

        .stSidebar {
            background: linear-gradient(180deg, #0e001f 0%, #180028 100%) !important;
            border-right: 2px solid var(--accent2) !important;
        }

        .stButton > button {
            background: linear-gradient(90deg, transparent, rgba(252,238,9,0.1), transparent) !important;
            color: var(--accent) !important;
            border: 1px solid var(--accent) !important;
            border-radius: 2px !important;
            font-family: var(--font-display) !important;
            font-size: 0.75rem !important;
            letter-spacing: 3px !important;
            text-transform: uppercase !important;
            box-shadow: 0 0 15px rgba(252,238,9,0.2), inset 0 0 15px rgba(252,238,9,0.05) !important;
            transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
        }

        .stButton > button:hover {
            background: linear-gradient(90deg, rgba(255,0,60,0.1), rgba(252,238,9,0.2), rgba(0,240,255,0.1)) !important;
            box-shadow: 0 0 25px rgba(252,238,9,0.5), 0 0 50px rgba(252,238,9,0.2) !important;
            transform: scale(1.05) !important;
        }

        div[data-testid="stMetric"] {
            background: linear-gradient(135deg, rgba(26,0,48,0.9), rgba(18,0,32,0.9)) !important;
            border: 1px solid rgba(252,238,9,0.3) !important;
            border-left: 3px solid var(--accent) !important;
            border-radius: 2px !important;
            padding: 12px !important;
            box-shadow: 0 0 20px rgba(252,238,9,0.05) !important;
            transition: all 0.3s ease !important;
        }

        div[data-testid="stMetric"]:hover {
            transform: translateY(-3px) !important;
            border-left-width: 5px !important;
        }

        .terminal-box {
            background: rgba(0,0,20,0.9);
            border: 1px solid rgba(0,240,255,0.4);
            border-top: 2px solid var(--accent);
            border-radius: 2px;
            padding: 16px;
            font-family: 'Rajdhani', monospace;
            font-size: 0.9rem;
            color: #00f0ff;
            box-shadow: 0 0 30px rgba(0,240,255,0.1), inset 0 0 30px rgba(0,0,0,0.5);
            white-space: pre-wrap;
            max-height: 400px;
            overflow-y: auto;
        }
        """
    },
    "Corporate Intel": {
        "icon": "🏢",
        "css": """
        @import url('https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;700&family=IBM+Plex+Sans:wght@300;400;600&display=swap');

        :root {
            --bg-primary: #f4f4f4;
            --bg-secondary: #e8e8e8;
            --bg-card: #ffffff;
            --accent: #0f62fe;
            --accent2: #da1e28;
            --accent3: #198038;
            --text-primary: #161616;
            --text-secondary: #525252;
            --border: #8d8d8d;
            --font-main: 'IBM Plex Sans', sans-serif;
            --font-display: 'IBM Plex Mono', monospace;
        }

        .stApp {
            background-color: var(--bg-primary) !important;
            font-family: var(--font-main) !important;
            color: var(--text-primary) !important;
        }

        h1, h2, h3, h4 {
            font-family: var(--font-display) !important;
            color: var(--text-primary) !important;
            letter-spacing: -0.5px !important;
            font-weight: 700 !important;
        }

        h1 {
            font-size: 2rem !important;
            border-bottom: 3px solid var(--accent) !important;
            padding-bottom: 8px !important;
        }

        .stSidebar {
            background-color: #161616 !important;
            border-right: none !important;
        }

        .stButton > button {
            background: var(--accent) !important;
            color: white !important;
            border: none !important;
            border-radius: 0 !important;
            font-family: var(--font-main) !important;
            font-weight: 600 !important;
            font-size: 0.875rem !important;
            padding: 10px 24px !important;
            transition: all 0.2s ease !important;
        }

        .stButton > button:hover {
            background: #0043ce !important;
            transform: translateY(-1px) !important;
            box-shadow: 0 4px 8px rgba(0,0,0,0.1) !important;
        }

        div[data-testid="stMetric"] {
            background: var(--bg-card) !important;
            border-left: 4px solid var(--accent) !important;
            border-radius: 0 !important;
            padding: 16px !important;
            box-shadow: 0 1px 3px rgba(0,0,0,0.1) !important;
            transition: all 0.2s ease !important;
        }

        div[data-testid="stMetric"]:hover {
            transform: translateY(-2px) !important;
            box-shadow: 0 4px 12px rgba(0,0,0,0.15) !important;
        }

        .terminal-box {
            background: #161616;
            border: none;
            padding: 16px;
            font-family: 'IBM Plex Mono', monospace;
            font-size: 0.825rem;
            color: #42be65;
            max-height: 400px;
            overflow-y: auto;
            white-space: pre-wrap;
        }
        """
    }
}

# ============================================================================
# IOC Extraction Engine
# ============================================================================

@dataclass
class IOC:
    type: str  # email, ip, domain, phone, url
    value: str
    source_url: str
    source_query: str
    context: str  # snippet où il a été trouvé
    confidence: float = 1.0
    additional_info: Dict = field(default_factory=dict)

@dataclass
class LeakSignature:
    pattern: str
    severity: str  # critical, high, medium, low
    description: str
    keywords: List[str]

class IOCExtractor:
    """Extraction d'indicateurs de compromission"""
    
    # Patterns regex pour différents types d'IOC
    EMAIL_PATTERN = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')
    IPV4_PATTERN = re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b')
    DOMAIN_PATTERN = re.compile(r'\b(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}\b')
    URL_PATTERN = re.compile(r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+[^\s]*')
    PHONE_PATTERN = re.compile(r'\b(?:\+?33|0)[1-9](?:[\s.-]?\d{2}){4}\b')
    
    # Patterns pour le mode "chasseur de fuites"
    LEAK_PATTERNS = [
        LeakSignature(r'(?i)(mot de passe|password|passwd|pwd)', 'high', 'Mot de passe exposé', ['password', 'mot de passe']),
        LeakSignature(r'(?i)(clé[ -]?api|api[ -]?key|apikey)', 'critical', 'Clé API exposée', ['api', 'key']),
        LeakSignature(r'(?i)(confidentiel|secret|top secret|classified)', 'high', 'Document confidentiel', ['secret', 'confidentiel']),
        LeakSignature(r'(?i)(token|jwt|bearer)', 'critical', 'Token d\'authentification', ['token', 'jwt']),
        LeakSignature(r'(?i)(compte bancaire|iban|rib|credit card|carte bancaire)', 'critical', 'Information bancaire', ['bank', 'iban', 'credit']),
        LeakSignature(r'(?i)(ssn|social security|numéro sécurité sociale|nir)', 'critical', 'Numéro de sécurité sociale', ['ssn', 'nir']),
        LeakSignature(r'(?i)(breach|data leak|fuite de données|compromission)', 'medium', 'Mention de fuite de données', ['leak', 'breach']),
        LeakSignature(r'(?i)(private|internal use only|do not share)', 'medium', 'Document interne', ['private', 'internal']),
        LeakSignature(r'(?i)(vulnerability|cve|exploit|zero day)', 'high', 'Vulnérabilité mentionnée', ['cve', 'exploit']),
    ]
    
    @classmethod
    def extract_emails(cls, text: str) -> List[str]:
        return list(set(cls.EMAIL_PATTERN.findall(text)))
    
    @classmethod
    def extract_ips(cls, text: str) -> List[str]:
        ips = cls.IPV4_PATTERN.findall(text)
        # Filtrer les IPs invalides
        valid_ips = []
        for ip in ips:
            parts = ip.split('.')
            if all(0 <= int(p) <= 255 for p in parts):
                valid_ips.append(ip)
        return list(set(valid_ips))
    
    @classmethod
    def extract_domains(cls, text: str) -> List[str]:
        domains = cls.DOMAIN_PATTERN.findall(text)
        # Filtrer les faux positifs (ex: des mots normaux)
        exclude = {'com', 'org', 'net', 'fr', 'www', 'http', 'https', 'html', 'php', 'asp'}
        valid_domains = []
        for d in domains:
            if len(d) > 4 and '.' in d and d.split('.')[-1] not in exclude:
                valid_domains.append(d.lower())
        return list(set(valid_domains))
    
    @classmethod
    def extract_urls(cls, text: str) -> List[str]:
        return list(set(cls.URL_PATTERN.findall(text)))
    
    @classmethod
    def extract_phones(cls, text: str) -> List[str]:
        phones = []
        for match in cls.PHONE_PATTERN.findall(text):
            try:
                phone_num = phonenumbers.parse(match, "FR")
                if phonenumbers.is_valid_number(phone_num):
                    phones.append(phonenumbers.format_number(phone_num, phonenumbers.PhoneNumberFormat.INTERNATIONAL))
            except:
                pass
        return list(set(phones))
    
    @classmethod
    def detect_leaks(cls, text: str, url: str, query: str) -> List[Dict]:
        """Détection de fuites potentielles"""
        leaks = []
        text_lower = text.lower()
        
        for signature in cls.LEAK_PATTERNS:
            if re.search(signature.pattern, text_lower):
                leaks.append({
                    'type': 'leak_signature',
                    'severity': signature.severity,
                    'description': signature.description,
                    'found_in': url,
                    'query': query,
                    'pattern': signature.pattern
                })
        
        # Détection de mots-clés sensibles supplémentaires
        sensitive_keywords = ['admin', 'root', 'config', '.env', 'database', 'sql', 'dump', 'backup']
        for kw in sensitive_keywords:
            if kw in text_lower:
                leaks.append({
                    'type': 'sensitive_keyword',
                    'severity': 'medium',
                    'description': f'Mot-clé sensible: {kw}',
                    'found_in': url,
                    'query': query,
                    'keyword': kw
                })
        
        return leaks
    
    @classmethod
    def extract_all_from_result(cls, result: Dict) -> Dict:
        """Extrait tous les IOC d'un résultat"""
        text_to_scan = f"{result.get('title', '')} {result.get('snippet', '')}"
        url = result.get('url', '')
        query = result.get('query', '')
        
        iocs = {
            'emails': cls.extract_emails(text_to_scan),
            'ips': cls.extract_ips(text_to_scan),
            'domains': cls.extract_domains(text_to_scan),
            'urls': cls.extract_urls(text_to_scan),
            'phones': cls.extract_phones(text_to_scan),
            'leaks': cls.detect_leaks(text_to_scan, url, query)
        }
        
        return iocs

# ============================================================================
# LOGIQUE MÉTIER (version améliorée)
# ============================================================================

@dataclass
class SearchResult:
    title: str
    url: str
    snippet: str
    source_engine: str
    query: str
    timestamp: str = ""
    rank: int = 0
    http_status: int = 0
    response_time: float = 0.0
    osint_relevance_score: float = 0.0
    iocs: Dict = field(default_factory=dict)
    leak_score: float = 0.0

    def __post_init__(self):
        if not self.timestamp:
            self.timestamp = datetime.now().isoformat()

    def to_dict(self) -> Dict:
        return asdict(self)


DEFAULT_CONFIG = {
    "profile_name": "default",
    "profile_description": "Configuration par défaut",
    "trusted_domains": {},
    "excluded_patterns": ['pinterest.', 'facebook.', 'twitter.', 'x.com', 'instagram.', 'tiktok.', 'amazon.', 'ebay.'],
    "excluded_extensions": ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.zip', '.exe'],
    "priority_keywords": [],
    "search_engines": ['DuckDuckGo'],
    "max_results_per_engine": 10,
    "max_concurrent_requests": 3,
    "delay_between_queries_min": 3,
    "delay_between_queries_max": 7,
    "delay_between_engines_min": 2,
    "delay_between_engines_max": 5,
    "http_timeout_total": 30,
    "http_timeout_connect": 10,
    "min_relevance_score": 50.0,
    "user_agents": ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'],
    "api_keys": {"brave": "", "serpapi": "", "searxng": "https://searx.be"},
    "output_formats": ["csv", "json"],
    "database_name": "osint_searches.db"
}


def calculate_osint_score(result: SearchResult, domain: str, config: Dict) -> float:
    trusted_domains = config.get("trusted_domains", {})
    priority_keywords = config.get("priority_keywords", [])
    score = 0.0
    score += trusted_domains.get(domain, 0) * 3
    content = (result.title + " " + result.snippet).lower()
    keyword_score = sum(5 for kw in priority_keywords if kw.lower() in content)
    score += min(keyword_score, 50)
    if any(x in domain for x in ['osint', 'geospatial', 'satellite', 'bellingcat', 'geoconfirmed']):
        score += 10
    if any(x in domain for x in ['.org', '.edu', '.gouv', '.gov']):
        score += 5
    
    # Bonus pour présence d'IOC
    if result.iocs:
        ioc_count = len(result.iocs.get('emails', [])) + len(result.iocs.get('ips', []))
        score += min(ioc_count * 5, 20)
        score += min(len(result.iocs.get('leaks', [])) * 10, 30)
    
    return min(score, 100.0)


def is_relevant(result: SearchResult, config: Dict) -> bool:
    url_lower = result.url.lower()
    for pattern in config.get("excluded_patterns", []):
        if pattern in url_lower:
            return False
    for ext in config.get("excluded_extensions", []):
        if url_lower.endswith(ext):
            return False
    return True


def normalize_url(url: str) -> str:
    if not url:
        return ""
    url = url.lower().strip()
    url = url.replace('https://', '').replace('http://', '').replace('www.', '').rstrip('/')
    return url


def extract_ddg_url(ddg_url: str) -> str:
    if not ddg_url:
        return ""
    if ddg_url.startswith('http') and 'duckduckgo.com' not in ddg_url:
        return ddg_url
    if 'uddg=' in ddg_url:
        try:
            parsed = urllib.parse.urlparse(ddg_url)
            params = urllib.parse.parse_qs(parsed.query)
            if 'uddg' in params:
                return urllib.parse.unquote(params['uddg'][0])
        except:
            pass
    return ""


DEFAULT_USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36',
]


async def search_duckduckgo(session, query: str, config: Dict, log_fn=None) -> List[SearchResult]:
    max_results = config.get("max_results_per_engine", 10)
    results = []
    try:
        encoded_query = urllib.parse.quote(query)
        url = f"https://html.duckduckgo.com/html/?q={encoded_query}"
        headers = {
            'User-Agent': random.choice(config.get("user_agents", DEFAULT_USER_AGENTS)),
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'fr-FR,fr;q=0.9,en;q=0.8',
        }
        start_time = time.time()
        async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=30)) as response:
            response_time = time.time() - start_time
            if response.status == 200:
                from bs4 import BeautifulSoup
                html = await response.text()
                soup = BeautifulSoup(html, 'html.parser')
                for idx, result_div in enumerate(soup.find_all('div', class_='result')[:max_results]):
                    title_tag = result_div.find('a', class_='result__a')
                    if title_tag:
                        title = title_tag.get_text(strip=True)
                        raw_link = title_tag.get('href', '')
                        real_url = extract_ddg_url(raw_link)
                        snippet_tag = result_div.find('a', class_='result__snippet')
                        snippet = snippet_tag.get_text(strip=True) if snippet_tag else ""
                        if real_url and real_url.startswith('http'):
                            r = SearchResult(title=title, url=real_url, snippet=snippet,
                                             source_engine="DuckDuckGo", query=query,
                                             rank=idx+1, http_status=200, response_time=response_time)
                            results.append(r)
                if log_fn:
                    log_fn(f"  ✓ DuckDuckGo: {len(results)} résultats")
            else:
                if log_fn:
                    log_fn(f"  ✗ DuckDuckGo: HTTP {response.status}")
    except Exception as e:
        if log_fn:
            log_fn(f"  ✗ DuckDuckGo erreur: {str(e)[:60]}")
    return results


async def search_searxng(session, query: str, config: Dict, log_fn=None) -> List[SearchResult]:
    max_results = config.get("max_results_per_engine", 10)
    results = []
    searxng_instance = config.get("api_keys", {}).get("searxng", "")
    if not searxng_instance:
        return results
    try:
        instance = searxng_instance.rstrip('/')
        params = {'q': query, 'format': 'json', 'engines': 'google,bing,duckduckgo', 'language': 'fr', 'pageno': 1}
        start_time = time.time()
        async with session.get(f"{instance}/search", params=params, timeout=aiohttp.ClientTimeout(total=30)) as response:
            response_time = time.time() - start_time
            if response.status == 200:
                data = await response.json()
                for idx, item in enumerate(data.get('results', [])[:max_results]):
                    r = SearchResult(title=item.get('title', ''), url=item.get('url', ''),
                                     snippet=item.get('content', ''),
                                     source_engine=f"SearXNG/{item.get('engine', '?')}",
                                     query=query, rank=idx+1, http_status=200, response_time=response_time)
                    results.append(r)
                if log_fn:
                    log_fn(f"  ✓ SearXNG: {len(results)} résultats")
    except Exception as e:
        if log_fn:
            log_fn(f"  ✗ SearXNG erreur: {str(e)[:60]}")
    return results


async def search_brave(session, query: str, config: Dict, log_fn=None) -> List[SearchResult]:
    max_results = config.get("max_results_per_engine", 10)
    results = []
    brave_key = config.get("api_keys", {}).get("brave", "")
    if not brave_key:
        if log_fn:
            log_fn("  ⚠ Brave: clé API manquante")
        return results
    try:
        headers = {'X-Subscription-Token': brave_key, 'Accept': 'application/json'}
        params = {'q': query, 'count': max_results}
        start_time = time.time()
        async with session.get('https://api.search.brave.com/res/v1/web/search',
                               headers=headers, params=params,
                               timeout=aiohttp.ClientTimeout(total=30)) as response:
            response_time = time.time() - start_time
            if response.status == 200:
                data = await response.json()
                for idx, item in enumerate(data.get('web', {}).get('results', [])[:max_results]):
                    r = SearchResult(title=item.get('title', ''), url=item.get('url', ''),
                                     snippet=item.get('description', ''),
                                     source_engine="Brave", query=query,
                                     rank=idx+1, http_status=200, response_time=response_time)
                    results.append(r)
                if log_fn:
                    log_fn(f"  ✓ Brave: {len(results)} résultats")
    except Exception as e:
        if log_fn:
            log_fn(f"  ✗ Brave erreur: {str(e)[:60]}")
    return results


async def search_qwant(session, query: str, config: Dict, log_fn=None) -> List[SearchResult]:
    max_results = config.get("max_results_per_engine", 10)
    results = []
    try:
        params = {'q': query, 't': 'web', 'count': max_results, 'locale': 'fr_FR'}
        headers = {'User-Agent': random.choice(config.get("user_agents", DEFAULT_USER_AGENTS))}
        start_time = time.time()
        async with session.get('https://api.qwant.com/v3/search/web',
                               params=params, headers=headers,
                               timeout=aiohttp.ClientTimeout(total=30)) as response:
            response_time = time.time() - start_time
            if response.status == 200:
                data = await response.json()
                items_raw = data.get('data', {}).get('result', {}).get('items', {}).get('mainline', [])
                idx = 0
                for item in items_raw:
                    if item.get('type') == 'web':
                        for web_result in item.get('items', []):
                            if idx >= max_results:
                                break
                            r = SearchResult(title=web_result.get('title', ''),
                                             url=web_result.get('url', ''),
                                             snippet=web_result.get('desc', ''),
                                             source_engine="Qwant", query=query,
                                             rank=idx+1, http_status=200, response_time=response_time)
                            results.append(r)
                            idx += 1
                if log_fn:
                    log_fn(f"  ✓ Qwant: {len(results)} résultats")
    except Exception as e:
        if log_fn:
            log_fn(f"  ✗ Qwant erreur: {str(e)[:60]}")
    return results


async def run_osint_search(queries: List[str], config: Dict,
                           selected_engines: List[str],
                           progress_bar, status_text, log_container,
                           extract_iocs: bool = True,
                           leak_hunter: bool = True) -> List[Dict]:
    """Moteur de recherche principal asynchrone avec IOC extraction"""
    all_results = []
    logs = []

    def log(msg):
        logs.append(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}")

    engine_map = {
        'DuckDuckGo': search_duckduckgo,
        'SearXNG': search_searxng,
        'Brave': search_brave,
        'Qwant': search_qwant,
    }

    connector = aiohttp.TCPConnector(limit=10, force_close=True)
    async with aiohttp.ClientSession(connector=connector) as session:
        seen_urls: Set[str] = set()
        total = len(queries)

        for i, query in enumerate(queries):
            log(f"━━━ [{i+1}/{total}] {query[:70]}")
            status_text.text(f"Requête {i+1}/{total}: {query[:60]}...")
            progress_bar.progress((i) / total)

            for engine_name in selected_engines:
                fn = engine_map.get(engine_name)
                if not fn:
                    continue

                log(f"  🔍 {engine_name}...")
                engine_results = await fn(session, query, config, log_fn=log)

                delay = random.uniform(
                    config.get("delay_between_engines_min", 1),
                    config.get("delay_between_engines_max", 3)
                )
                await asyncio.sleep(delay)

                for r in engine_results:
                    norm = normalize_url(r.url)
                    if norm and norm not in seen_urls and is_relevant(r, config):
                        seen_urls.add(norm)
                        
                        # Extraction IOC
                        if extract_iocs or leak_hunter:
                            r.iocs = IOCExtractor.extract_all_from_result(r.to_dict())
                            
                            # Calcul du leak_score
                            if leak_hunter and r.iocs.get('leaks'):
                                r.leak_score = min(len(r.iocs['leaks']) * 20, 100)
                        
                        domain = urllib.parse.urlparse(r.url).netloc.replace('www.', '')
                        score = calculate_osint_score(r, domain, config)
                        r.osint_relevance_score = score
                        
                        # Alerte pour fuites critiques
                        if leak_hunter and r.leak_score >= 40:
                            log(f"  ⚠️⚠️ FUITE CRITIQUE détectée dans {r.url[:60]}... (score: {r.leak_score})")
                        
                        all_results.append(r.to_dict())

            log_container.markdown(
                f'<div class="terminal-box">' + '\n'.join(logs[-40:]) + '</div>',
                unsafe_allow_html=True
            )

            if i < total - 1:
                delay = random.uniform(
                    config.get("delay_between_queries_min", 2),
                    config.get("delay_between_queries_max", 5)
                )
                await asyncio.sleep(delay)

        progress_bar.progress(1.0)
        status_text.text(f"✅ Terminé — {len(all_results)} résultats collectés")

    return all_results


def parse_queries(text: str) -> List[str]:
    queries = []
    for line in text.splitlines():
        line = line.strip()
        if line and not line.startswith('#'):
            queries.append(line)
    return queries


def results_to_csv(results: List[Dict]) -> str:
    if not results:
        return ""
    output = io.StringIO()
    fieldnames = ['query', 'title', 'url', 'snippet', 'source_engine', 'rank', 'osint_relevance_score', 'leak_score', 'timestamp']
    writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction='ignore')
    writer.writeheader()
    writer.writerows(results)
    return output.getvalue()


# ============================================================================
# APP STREAMLIT
# ============================================================================
def get_theme_names():
    return list(THEMES.keys())


def apply_theme(theme_name: str):
    theme = THEMES.get(theme_name, THEMES["Terminal Rétro"])
    st.markdown(f"<style>{theme['css']}</style>", unsafe_allow_html=True)


def render_banner(theme_name: str):
    theme = THEMES[theme_name]
    if theme_name == "Terminal Rétro":
        st.markdown("""
        <div style="border:1px solid #00ff41; padding:20px; margin-bottom:20px; 
                    box-shadow: 0 0 20px rgba(0,255,65,0.2);">
        <pre style="color:#00ff41; font-family:'Share Tech Mono',monospace; margin:0; font-size:0.85rem;">
╔══════════════════════════════════════════════════════════════╗
║   ██████╗  ██████╗ ██████╗ ██╗  ██╗███████╗██████╗          ║
║   ██╔══██╗██╔═══██╗██╔══██╗██║ ██╔╝██╔════╝██╔══██╗         ║
║   ██║  ██║██║   ██║██████╔╝█████╔╝ █████╗  ██████╔╝         ║
║   ██║  ██║██║   ██║██╔══██╗██╔═██╗ ██╔══╝  ██╔══██╗         ║
║   ██████╔╝╚██████╔╝██║  ██║██║  ██╗███████╗██║  ██║         ║
║   ╚═════╝  ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═╝╚══════╝╚═╝  ╚═╝  PRO   ║
║         OSINT v5.0 - HUNTER EDITION                        ║
╚══════════════════════════════════════════════════════════════╝
        </pre>
        </div>
        """, unsafe_allow_html=True)
    elif theme_name == "CyberPunk 2077":
        st.markdown("""
        <div style="border:1px solid rgba(252,238,9,0.4); border-top:3px solid #fcee09; 
                    padding:24px; margin-bottom:20px;
                    background:linear-gradient(135deg,rgba(26,0,48,0.8),rgba(10,0,20,0.9));
                    box-shadow:0 0 40px rgba(252,238,9,0.1);">
        <h1 style="margin:0; font-family:'Orbitron',monospace; font-size:2rem; 
                   color:#fcee09; text-shadow:0 0 15px #fcee09,-2px 2px 0 #ff003c;
                   letter-spacing:6px;">DORKER<span style="color:#ff003c;">_</span>PRO</h1>
        <p style="margin:8px 0 0 0; font-family:'Rajdhani',sans-serif; color:#9b7fd4; 
                  letter-spacing:4px; font-size:0.85rem;">
            ⚡ OSINT v5.0 // HUNTER EDITION // IOC EXTRACTION ACTIVE
        </p>
        </div>
        """, unsafe_allow_html=True)
    else:
        st.markdown("""
        <div style="border-left:4px solid #0f62fe; padding:16px 24px; margin-bottom:20px; background:#fff; box-shadow:0 1px 3px rgba(0,0,0,0.1);">
        <h1 style="margin:0; font-family:'IBM Plex Mono',monospace; color:#161616; font-size:1.75rem;">Dorker Pro OSINT v5.0</h1>
        <p style="margin:4px 0 0; font-family:'IBM Plex Sans',sans-serif; color:#525252; font-size:0.875rem;">
            Intelligence Collection Platform — avec extraction automatique d'IOC
        </p>
        </div>
        """, unsafe_allow_html=True)


def render_ioc_dashboard(iocs_data: List[Dict]):
    """Affiche le tableau de bord des IOC"""
    if not iocs_data:
        return
    
    all_emails = []
    all_ips = []
    all_domains = []
    all_phones = []
    all_leaks = []
    
    for item in iocs_data:
        iocs = item.get('iocs', {})
        all_emails.extend(iocs.get('emails', []))
        all_ips.extend(iocs.get('ips', []))
        all_domains.extend(iocs.get('domains', []))
        all_phones.extend(iocs.get('phones', []))
        all_leaks.extend(iocs.get('leaks', []))
    
    st.markdown("### 🎯 Indicateurs de Compromission (IOC) extraits")
    
    col1, col2, col3, col4, col5 = st.columns(5)
    with col1:
        st.metric("📧 Emails", len(set(all_emails)))
    with col2:
        st.metric("🌐 IPs", len(set(all_ips)))
    with col3:
        st.metric("🏠 Domaines", len(set(all_domains)))
    with col4:
        st.metric("📞 Téléphones", len(set(all_phones)))
    with col5:
        critical_leaks = sum(1 for l in all_leaks if l.get('severity') in ['critical', 'high'])
        st.metric("⚠️ Fuites critiques", critical_leaks, delta_color="off")
    
    # Affichage détaillé
    tabs = st.tabs(["📧 Emails", "🌐 IPs", "🏠 Domaines", "📞 Téléphones", "⚠️ Fuites détectées"])
    
    with tabs[0]:
        if all_emails:
            for email in sorted(set(all_emails)):
                st.markdown(f'<div class="ioc-email">📧 {email}</div>', unsafe_allow_html=True)
        else:
            st.info("Aucun email détecté")
    
    with tabs[1]:
        if all_ips:
            for ip in sorted(set(all_ips)):
                st.markdown(f'<div class="ioc-ip">🌐 {ip}</div>', unsafe_allow_html=True)
        else:
            st.info("Aucune IP détectée")
    
    with tabs[2]:
        if all_domains:
            domain_counts = Counter(all_domains)
            for domain, count in domain_counts.most_common(30):
                st.markdown(f"🏠 {domain} *({count} occurrence(s))*")
        else:
            st.info("Aucun domaine détecté")
    
    with tabs[3]:
        if all_phones:
            for phone in sorted(set(all_phones)):
                st.markdown(f"📞 {phone}")
        else:
            st.info("Aucun numéro de téléphone détecté")
    
    with tabs[4]:
        if all_leaks:
            for leak in all_leaks:
                severity_color = {
                    'critical': '🔴 CRITIQUE',
                    'high': '🟠 HAUTE',
                    'medium': '🟡 MOYENNE',
                    'low': '🔵 BASSE'
                }.get(leak.get('severity'), '⚪ INCONNUE')
                
                st.markdown(f"""
                <div class="leak-alert">
                    <b>{severity_color}</b> - {leak.get('description', '')}<br>
                    <small>📍 {leak.get('found_in', '')[:80]}...<br>
                    🔍 Requête: {leak.get('query', '')}</small>
                </div>
                """, unsafe_allow_html=True)
        else:
            st.info("Aucune fuite détectée")


def main():
    st.set_page_config(
        page_title="Dorker Pro OSINT v5.0 - Hunter Edition",
        page_icon="🔍",
        layout="wide",
        initial_sidebar_state="expanded"
    )

    # ---- Sélecteur de thème et mode compact ----
    with st.sidebar:
        st.markdown("## 🎨 Interface")
        theme_name = st.selectbox(
            "Thème",
            options=get_theme_names(),
            format_func=lambda x: f"{THEMES[x]['icon']} {x}",
            key="theme_selector"
        )
        
        compact_mode = st.checkbox("📐 Mode compact", value=False, help="Affichage plus dense pour les pros")
        
        st.markdown("---")
        st.markdown("## 🕵️ Mode Chasseur de fuites")
        leak_hunter_mode = st.checkbox("🔴 Activer Chasseur de fuites", value=True, 
                                       help="Détection automatique de mots-clés sensibles (mots de passe, clés API, etc.)")
        extract_iocs = st.checkbox("🎯 Extraction IOC", value=True,
                                   help="Extraction automatique d'emails, IPs, domaines, téléphones")

    apply_theme(theme_name)
    
    if compact_mode:
        st.markdown('<div class="compact-mode">', unsafe_allow_html=True)

    # ---- Sidebar — Configuration ----
    with st.sidebar:
        st.markdown("---")
        st.markdown("## ⚙️ Configuration")

        config_file = st.file_uploader(
            "Profil JSON",
            type=["json"],
            help="Chargez votre fichier de configuration"
        )

        config = DEFAULT_CONFIG.copy()
        profile_name = "default"

        if config_file is not None:
            try:
                loaded = json.load(config_file)
                config = {**DEFAULT_CONFIG, **loaded}
                profile_name = config.get("profile_name", config_file.name)
                st.success(f"✅ Profil: **{profile_name}**")
                if config.get("profile_description"):
                    st.caption(config["profile_description"])
            except Exception as e:
                st.error(f"❌ Erreur JSON: {e}")

        st.markdown("---")
        st.markdown("## 🔧 Paramètres")

        selected_engines = st.multiselect(
            "Moteurs de recherche",
            options=["DuckDuckGo", "SearXNG", "Brave", "Qwant"],
            default=config.get("search_engines", ["DuckDuckGo"])
        )

        max_results = st.number_input(
            "Résultats max / moteur",
            min_value=1, max_value=50,
            value=config.get("max_results_per_engine", 10)
        )
        config["max_results_per_engine"] = max_results

        min_score = st.slider(
            "Score OSINT minimum",
            min_value=0, max_value=100,
            value=int(config.get("min_relevance_score", 45))
        )
        config["min_relevance_score"] = float(min_score)

        st.markdown("---")
        st.markdown("## 🔑 Clés API")
        brave_key = st.text_input("Brave API Key", value=config.get("api_keys", {}).get("brave", ""), type="password")
        serpapi_key = st.text_input("SerpAPI Key", value=config.get("api_keys", {}).get("serpapi", ""), type="password")
        searxng_url = st.text_input("SearXNG URL", value=config.get("api_keys", {}).get("searxng", "https://searx.be"))
        config["api_keys"] = {"brave": brave_key, "serpapi": serpapi_key, "searxng": searxng_url}

        st.markdown("---")
        st.markdown("## ⏱️ Délais (sec)")
        col1, col2 = st.columns(2)
        with col1:
            q_min = st.number_input("Q. min", min_value=0, max_value=30, value=config.get("delay_between_queries_min", 2))
            e_min = st.number_input("E. min", min_value=0, max_value=15, value=config.get("delay_between_engines_min", 1))
        with col2:
            q_max = st.number_input("Q. max", min_value=0, max_value=60, value=config.get("delay_between_queries_max", 5))
            e_max = st.number_input("E. max", min_value=0, max_value=30, value=config.get("delay_between_engines_max", 3))
        config.update({
            "delay_between_queries_min": q_min,
            "delay_between_queries_max": q_max,
            "delay_between_engines_min": e_min,
            "delay_between_engines_max": e_max,
        })

    # ---- Zone principale ----
    render_banner(theme_name)

    tabs = st.tabs(["📋 Requêtes", "🚀 Lancer", "📊 Résultats", "🔍 IOC Dashboard", "⚙️ Profil JSON"])

    # ---- Tab 1 : Requêtes ----
    with tabs[0]:
        st.markdown("### Fichier de requêtes")
        queries_file = st.file_uploader(
            "Charger un fichier .txt",
            type=["txt"],
            help="Lignes de requêtes (# pour commenter)"
        )

        default_queries = """# Exemples de requêtes
# Mode OSINT standard
photocatalysis mechanism review 2024
green chemistry sustainable synthesis

# Mode chasseur de fuites (exemples sensibles)
"mot de passe" "fichier de configuration" site:github.com
api_key secret token
"confidentiel" "interne" filetype:pdf
carte bancaire "site:pastebin.com"
"""

        if queries_file is not None:
            queries_text = queries_file.read().decode("utf-8")
            st.success(f"✅ Fichier chargé: {queries_file.name}")
        else:
            queries_text = default_queries

        queries_edited = st.text_area(
            "Éditeur de requêtes",
            value=queries_text,
            height=300,
            help="Modifiez directement ici. Les lignes commençant par # sont ignorées."
        )

        parsed = parse_queries(queries_edited)
        st.info(f"**{len(parsed)}** requête(s) active(s) détectée(s)")

        if parsed:
            with st.expander("Aperçu des requêtes actives"):
                for i, q in enumerate(parsed, 1):
                    st.markdown(f"`{i:02d}.` {q}")

    # ---- Tab 2 : Lancer ----
    with tabs[1]:
        st.markdown("### Lancement de la veille OSINT")

        if not selected_engines:
            st.warning("⚠️ Sélectionnez au moins un moteur de recherche dans la sidebar.")
        else:
            queries_for_run = parse_queries(queries_edited if 'queries_edited' in dir() else default_queries)

            col1, col2, col3, col4 = st.columns(4)
            col1.metric("Requêtes", len(queries_for_run))
            col2.metric("Moteurs", len(selected_engines))
            col3.metric("Max résultats/moteur", max_results)
            col4.metric("Mode Chasseur", "🔴 ON" if leak_hunter_mode else "⚪ OFF")

            st.markdown("---")

            dry_run = st.checkbox("🧪 Mode DRY-RUN (pas de requêtes réelles)", value=False)

            launch_btn = st.button("🚀 LANCER LA VEILLE", use_container_width=True)

            if launch_btn:
                if not queries_for_run:
                    st.error("❌ Aucune requête active. Ajoutez des requêtes dans l'onglet Requêtes.")
                elif dry_run:
                    st.info("🧪 Mode DRY-RUN — simulation uniquement")
                    with st.expander("Résumé de ce qui serait exécuté"):
                        st.json({
                            "requetes": queries_for_run,
                            "moteurs": selected_engines,
                            "leak_hunter": leak_hunter_mode,
                            "extract_iocs": extract_iocs,
                            "config": {k: v for k, v in config.items() if k != "user_agents"}
                        })
                else:
                    st.markdown("---")
                    progress_bar = st.progress(0)
                    status_text = st.empty()
                    st.markdown("**Journal d'exécution:**")
                    log_container = st.empty()
                    log_container.markdown(
                        '<div class="terminal-box">Initialisation...</div>',
                        unsafe_allow_html=True
                    )

                    with st.spinner("Veille OSINT en cours..."):
                        results = asyncio.run(run_osint_search(
                            queries=queries_for_run,
                            config=config,
                            selected_engines=selected_engines,
                            progress_bar=progress_bar,
                            status_text=status_text,
                            log_container=log_container,
                            extract_iocs=extract_iocs,
                            leak_hunter=leak_hunter_mode
                        ))

                    st.session_state["results"] = results
                    st.session_state["run_complete"] = True
                    st.success(f"✅ Veille terminée — **{len(results)}** résultats collectés")
                    st.balloons()

    # ---- Tab 3 : Résultats ----
    with tabs[2]:
        st.markdown("### Résultats de la veille")

        results = st.session_state.get("results", [])

        if not results:
            st.info("Aucun résultat disponible. Lancez une veille dans l'onglet **Lancer**.")
        else:
            df = pd.DataFrame(results)
            
            # Score moyen avec code couleur
            avg_score = df['osint_relevance_score'].mean()
            score_class = "score-high" if avg_score >= 70 else "score-med" if avg_score >= 40 else "score-low"
            
            high_value = df[df["osint_relevance_score"] >= config.get("min_relevance_score", 45)]
            critical_leaks = sum(1 for r in results if r.get('leak_score', 0) >= 60)

            col1, col2, col3, col4, col5 = st.columns(5)
            col1.metric("Total résultats", len(df))
            col2.metric("URLs uniques", df["url"].nunique())
            col3.metric("Haute valeur", len(high_value))
            with col4:
                st.markdown(f'<div class="{score_class}">Score moyen<br>{avg_score:.1f}/100</div>', unsafe_allow_html=True)
            col5.metric("⚠️ Fuites critiques", critical_leaks, delta_color="inverse")

            st.markdown("---")

            col_a, col_b = st.columns([2, 1])
            with col_a:
                filter_engine = st.multiselect(
                    "Filtrer par moteur",
                    options=list(df["source_engine"].unique()),
                    default=list(df["source_engine"].unique())
                )
            with col_b:
                filter_score = st.slider("Score minimum", 0, 100, 0)

            filtered_df = df[
                df["source_engine"].isin(filter_engine) &
                (df["osint_relevance_score"] >= filter_score)
            ].sort_values("osint_relevance_score", ascending=False)

            st.markdown(f"**{len(filtered_df)}** résultats affichés")

            display_cols = ["osint_relevance_score", "title", "url", "source_engine", "query", "leak_score"]
            st.dataframe(
                filtered_df[display_cols].rename(columns={
                    "osint_relevance_score": "Score",
                    "title": "Titre",
                    "url": "URL",
                    "source_engine": "Moteur",
                    "query": "Requête",
                    "leak_score": "Fuites"
                }),
                use_container_width=True,
                height=400
            )

    # ---- Tab 4 : IOC Dashboard ----
    with tabs[3]:
        results = st.session_state.get("results", [])
        if not results:
            st.info("Aucun résultat disponible. Lancez une veille pour voir les IOC extraits.")
        else:
            render_ioc_dashboard(results)

    # ---- Tab 5 : Profil JSON ----
    with tabs[4]:
        st.markdown("### Profil de configuration actif")

        display_config = {k: v for k, v in config.items() if k not in ["user_agents"]}
        config_str = json.dumps(display_config, indent=2, ensure_ascii=False)

        st.text_area("Configuration JSON", value=config_str, height=500)

        st.download_button(
            label="💾 Télécharger ce profil JSON",
            data=config_str,
            file_name=f"profile_{profile_name}_{datetime.now().strftime('%Y%m%d')}.json",
            mime="application/json"
        )

    if compact_mode:
        st.markdown('</div>', unsafe_allow_html=True)


if __name__ == "__main__":
    main()