cdp_scraper.py

"""
CDP Scraper — Anti-Bot Web Scraping via Chrome DevTools Protocol.

Bypasses Cloudflare, DataDome, PerimeterX, and other anti-bot systems
by controlling the user's real Chrome browser via CDP WebSocket.
No Selenium, no chromedriver, no automation flags — just raw CDP.

Usage:
    scraper = CDPScraper(output_dir="./output")
    scraper.scrape_pages(
        base_url="https://example.com/thread",
        url_pattern="{base}-{page}.html",
        total_pages=10,
        extract_js="document.body.innerText",
    )

    # Convenience: scrape one page and get the text back directly
    text = scraper.scrape_single_page("https://example.com/article")
"""

import argparse
import json
import os
import platform
import shutil
import signal
import subprocess
import sys
import time
import urllib.request
from typing import List, Optional


# ── Chrome auto-detection ─────────────────────────────────────────────────────

def _find_chrome() -> str:
    """Auto-detect the Chrome executable path for the current platform.

    Returns:
        Absolute path to the Chrome binary.

    Raises:
        FileNotFoundError: If Chrome cannot be located on this system.
    """
    system = platform.system()
    candidates: List[str] = []

    if system == "Windows":
        prog = os.environ.get("PROGRAMFILES", r"C:\Program Files")
        prog86 = os.environ.get("PROGRAMFILES(X86)", r"C:\Program Files (x86)")
        local = os.environ.get("LOCALAPPDATA", "")
        candidates = [
            os.path.join(prog, r"Google\Chrome\Application\chrome.exe"),
            os.path.join(prog86, r"Google\Chrome\Application\chrome.exe"),
            os.path.join(local, r"Google\Chrome\Application\chrome.exe"),
        ]
    elif system == "Darwin":  # macOS
        candidates = [
            "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
            os.path.expanduser(
                "~/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
            ),
        ]
    else:  # Linux / BSD
        candidates = [
            "/usr/bin/google-chrome",
            "/usr/bin/google-chrome-stable",
            "/usr/bin/chromium",
            "/usr/bin/chromium-browser",
            "/snap/bin/chromium",
        ]

    # Also check PATH for common binary names
    for name in ("google-chrome", "google-chrome-stable", "chromium", "chromium-browser"):
        found = shutil.which(name)
        if found:
            candidates.insert(0, found)

    for path in candidates:
        if path and os.path.isfile(path):
            return path

    raise FileNotFoundError(
        "Chrome not found. Install Google Chrome or set the CHROME_PATH "
        f"environment variable.\nSearched locations: {candidates}"
    )


DEBUG_PORT: int = int(os.environ.get("CDP_PORT", "9222"))

# Path for persisted Cloudflare clearance cookie cache
_COOKIE_CACHE_FILE = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), ".cf_cookie_cache.json"
)


# ── Default JS extractor for forum-style pages ────────────────────────────────

DEFAULT_EXTRACT_JS = r"""
(function() {
    var posts = document.querySelectorAll('.postcontent, .postbody, [id^="post_message_"]');
    var result = '';
    posts.forEach(function(p) {
        var container = p.closest('.postbitlegacy, .postbit, [id^="post"]');
        var authorEl = container ? container.querySelector('.username, .bigusername') : null;
        var author = authorEl ? authorEl.textContent.trim() : '?';
        var dateEl = container ? container.querySelector('.postdate, .date, .thead') : null;
        var date = dateEl ? dateEl.textContent.trim() : '';
        result += '--- ' + author + ' (' + date + ') ---\n';
        result += p.innerText + '\n\n';
    });
    if (!posts.length) result = document.body.innerText;
    return result;
})()
"""

# Words that appear in anti-bot challenge page titles
_CHALLENGE_WORDS = ("moment", "challenge", "checking", "just a", "ddos-guard")


# ── Cookie cache helpers ──────────────────────────────────────────────────────

def _load_cookie_cache() -> dict:
    """Load the persisted Cloudflare cookie cache from disk.

    Returns:
        Dict mapping domain -> cf_clearance value, or empty dict.
    """
    try:
        if os.path.isfile(_COOKIE_CACHE_FILE):
            with open(_COOKIE_CACHE_FILE, "r", encoding="utf-8") as fh:
                return json.load(fh)
    except Exception:
        pass
    return {}


def _save_cookie_cache(cache: dict) -> None:
    """Persist the cookie cache to disk.

    Args:
        cache: Dict mapping domain -> cf_clearance value.
    """
    try:
        with open(_COOKIE_CACHE_FILE, "w", encoding="utf-8") as fh:
            json.dump(cache, fh, indent=2)
    except Exception:
        pass


def _domain_from_url(url: str) -> str:
    """Extract the bare domain from a URL (no scheme, no path).

    Args:
        url: Full URL string.

    Returns:
        Domain string, e.g. ``"forum.example.com"``.
    """
    try:
        # Strip scheme
        without_scheme = url.split("://", 1)[-1]
        # Strip path / query
        return without_scheme.split("/")[0].split("?")[0].split("#")[0]
    except Exception:
        return url


# ── ETA tracker ──────────────────────────────────────────────────────────────

class _ETATracker:
    """Lightweight tracker for estimating time-to-completion.

    Args:
        total: Total number of work items.
    """

    def __init__(self, total: int) -> None:
        self.total = total
        self._start = time.monotonic()
        self._completed = 0

    def tick(self) -> None:
        """Record one completed item."""
        self._completed += 1

    def eta_str(self) -> str:
        """Return a human-readable ETA string like ``"ETA 1m 23s"`` or ``"ETA --"``."""
        if self._completed == 0:
            return "ETA --"
        elapsed = time.monotonic() - self._start
        rate = self._completed / elapsed  # items per second
        remaining = self.total - self._completed
        if rate <= 0:
            return "ETA --"
        secs = int(remaining / rate)
        if secs >= 60:
            return f"ETA {secs // 60}m {secs % 60:02d}s"
        return f"ETA {secs}s"


# ── Core scraper class ────────────────────────────────────────────────────────

class CDPScraper:
    """Web scraper that controls Chrome via the DevTools Protocol (CDP).

    Bypasses anti-bot protections by using the user's real Chrome binary
    with a dedicated profile directory — no Selenium, no automation flags,
    no modified browser fingerprint.

    Args:
        output_dir: Directory where scraped pages are saved.
        debug_port: CDP remote debugging port (default: 9222, or $CDP_PORT).
        chrome_path: Path to the Chrome executable. Auto-detected if omitted
                     (also honoured via the $CHROME_PATH env variable).
        headless: Run Chrome in headless mode. Faster for sites without
                  anti-bot, but detectable on sites that check for a display.
        show_progress: Show a rich progress bar with percentage and ETA.
        max_retries: Number of retry attempts per page on transient failure.
        cache_cookies: Persist Cloudflare cf_clearance cookies between runs
                       so the challenge only needs solving once per domain.
    """

    def __init__(
        self,
        output_dir: str = "./output",
        debug_port: int = DEBUG_PORT,
        chrome_path: Optional[str] = None,
        headless: bool = False,
        show_progress: bool = False,
        max_retries: int = 3,
        cache_cookies: bool = True,
    ) -> None:
        self.output_dir = output_dir
        self.debug_port = debug_port
        self.chrome_path = chrome_path or self._resolve_chrome()
        self.headless = headless
        self.show_progress = show_progress
        self.max_retries = max_retries
        self.cache_cookies = cache_cookies
        self.chrome_proc: Optional[subprocess.Popen] = None
        os.makedirs(output_dir, exist_ok=True)
        self._register_signal_handlers()

    # ── Setup helpers ─────────────────────────────────────────────────────────

    @staticmethod
    def _resolve_chrome() -> str:
        """Return Chrome path from $CHROME_PATH or auto-detection."""
        env_path = os.environ.get("CHROME_PATH", "").strip()
        if env_path and os.path.isfile(env_path):
            return env_path
        return _find_chrome()

    def _register_signal_handlers(self) -> None:
        """Register SIGINT/SIGTERM handlers so Chrome is cleanly shut down."""

        def _handler(sig, frame):
            print("\nInterrupt received — shutting down Chrome...")
            self.close()
            sys.exit(0)

        signal.signal(signal.SIGINT, _handler)
        if hasattr(signal, "SIGTERM"):
            signal.signal(signal.SIGTERM, _handler)

    # ── CDP communication ─────────────────────────────────────────────────────

    def _get_tabs(self) -> List[dict]:
        """Fetch the list of open Chrome tabs via the CDP HTTP endpoint.

        Returns:
            List of tab descriptor dicts, or an empty list on failure.
        """
        try:
            req = urllib.request.urlopen(
                f"http://localhost:{self.debug_port}/json", timeout=5
            )
            return json.loads(req.read())
        except Exception:
            return []

    def _evaluate_js(self, ws_url: str, expression: str) -> Optional[str]:
        """Execute JavaScript in the given tab and return its string value.

        Args:
            ws_url: WebSocket debugger URL for the target tab.
            expression: JavaScript expression to evaluate. Must return a value
                        that can be serialised to a string.

        Returns:
            The return value as a Python string, or None on failure.

        Raises:
            ImportError: If the ``websocket-client`` package is not installed.
        """
        try:
            import websocket  # type: ignore[import]
        except ImportError as exc:
            raise ImportError(
                "websocket-client is required: pip install websocket-client"
            ) from exc

        ws = websocket.create_connection(ws_url, timeout=30)
        try:
            payload = json.dumps({
                "id": 1,
                "method": "Runtime.evaluate",
                "params": {"expression": expression, "returnByValue": True},
            })
            ws.send(payload)
            result = json.loads(ws.recv())
            return result.get("result", {}).get("result", {}).get("value")
        except Exception:
            return None
        finally:
            ws.close()

    def _navigate(self, ws_url: str, url: str) -> None:
        """Navigate the given Chrome tab to a URL via CDP Page.navigate.

        Args:
            ws_url: WebSocket debugger URL for the target tab.
            url: Destination URL.

        Raises:
            ImportError: If the ``websocket-client`` package is not installed.
        """
        try:
            import websocket  # type: ignore[import]
        except ImportError as exc:
            raise ImportError(
                "websocket-client is required: pip install websocket-client"
            ) from exc

        ws = websocket.create_connection(ws_url, timeout=30)
        try:
            payload = json.dumps({
                "id": 1,
                "method": "Page.navigate",
                "params": {"url": url},
            })
            ws.send(payload)
            ws.recv()
        finally:
            ws.close()

    # ── Cookie management ─────────────────────────────────────────────────────

    def _read_cf_clearance(self, ws_url: str) -> Optional[str]:
        """Read the current cf_clearance cookie value from the active tab.

        Args:
            ws_url: WebSocket debugger URL for the target tab.

        Returns:
            Cookie value string, or None if not present.
        """
        js = (
            "(function(){"
            "var m=document.cookie.match(/cf_clearance=([^;]+)/);"
            "return m?m[1]:null;"
            "})()"
        )
        return self._evaluate_js(ws_url, js)

    def _inject_cf_clearance(self, ws_url: str, value: str) -> None:
        """Inject a cached cf_clearance cookie into the active tab.

        Args:
            ws_url: WebSocket debugger URL for the target tab.
            value: Cookie value to inject.
        """
        exp = int(time.time()) + 60 * 60 * 24  # 24 h from now
        js = (
            f"document.cookie='cf_clearance={value};"
            f"expires={exp};path=/;SameSite=None;Secure';"
        )
        self._evaluate_js(ws_url, js)

    def _maybe_restore_cookies(self, ws_url: str, domain: str) -> bool:
        """Restore cached Cloudflare cookies for a domain if available.

        Args:
            ws_url: WebSocket debugger URL.
            domain: Domain to look up in the cache.

        Returns:
            True if a cached cookie was injected.
        """
        if not self.cache_cookies:
            return False
        cache = _load_cookie_cache()
        value = cache.get(domain)
        if value:
            print(f"  Restoring cached cf_clearance for {domain}")
            self._inject_cf_clearance(ws_url, value)
            return True
        return False

    def _maybe_save_cookies(self, ws_url: str, domain: str) -> None:
        """Save the current cf_clearance cookie to the cache for future runs.

        Args:
            ws_url: WebSocket debugger URL.
            domain: Domain key to store the cookie under.
        """
        if not self.cache_cookies:
            return
        value = self._read_cf_clearance(ws_url)
        if value:
            cache = _load_cookie_cache()
            if cache.get(domain) != value:
                cache[domain] = value
                _save_cookie_cache(cache)
                print(f"  cf_clearance saved for {domain} (future runs skip the challenge)")

    # ── Chrome lifecycle ──────────────────────────────────────────────────────

    def _launch_chrome(self, start_url: str) -> None:
        """Start Chrome with CDP enabled, pointed at start_url.

        A dedicated user-data directory inside output_dir is used so that
        the scraper does not interfere with the user's default Chrome profile.

        Args:
            start_url: URL to open on launch.
        """
        user_data = os.path.join(self.output_dir, "_chrome_profile")
        os.makedirs(user_data, exist_ok=True)

        cmd = [
            self.chrome_path,
            f"--remote-debugging-port={self.debug_port}",
            "--remote-allow-origins=*",
            f"--user-data-dir={user_data}",
        ]
        if self.headless:
            cmd.append("--headless=new")
        cmd.append(start_url)

        self.chrome_proc = subprocess.Popen(cmd)

    def _wait_for_page(
        self, keyword: str = "", timeout: int = 300
    ) -> Optional[dict]:
        """Poll Chrome until a tab is ready (past any anti-bot challenge).

        Args:
            keyword: If provided, wait until a tab title contains this string.
                     If empty, wait until no tab appears to show a challenge.
            timeout: Maximum seconds to wait before giving up.

        Returns:
            Tab descriptor dict for the first matching tab, or None on timeout.
        """
        for attempt in range(timeout // 5):
            tabs = self._get_tabs()
            if tabs:
                for tab in tabs:
                    title = tab.get("title", "").lower()
                    if keyword and keyword.lower() in title:
                        return tab
                    if not keyword and not any(w in title for w in _CHALLENGE_WORDS):
                        return tab
                if attempt % 6 == 0:
                    elapsed = attempt * 5
                    current = tabs[0].get("title", "?")[:60]
                    print(f"  Waiting for page... ({elapsed}s) title='{current}'")
            time.sleep(5)

        tabs = self._get_tabs()
        return tabs[0] if tabs else None

    # ── Progress display ──────────────────────────────────────────────────────

    @staticmethod
    def _progress_bar(current: int, total: int, eta: str = "", width: int = 36) -> str:
        """Return a formatted progress bar string with percentage and ETA.

        Args:
            current: Current step (1-based).
            total: Total number of steps.
            eta: ETA string to append, e.g. ``"ETA 1m 23s"``.
            width: Number of characters for the bar fill area.

        Returns:
            String like ``[####----] 40% (2/5)  ETA 30s``.
        """
        filled = int(width * current / total)
        bar = "#" * filled + "-" * (width - filled)
        pct = int(100 * current / total)
        eta_part = f"  {eta}" if eta else ""
        return f"[{bar}] {pct:3d}% ({current}/{total}){eta_part}"

    # ── Public API ────────────────────────────────────────────────────────────

    def scrape_single_page(
        self,
        url: str,
        extract_js: str = "document.body.innerText",
        load_wait: float = 2.0,
    ) -> str:
        """Scrape a single page and return its text directly.

        Convenience wrapper around :meth:`scrape_single` that handles launch,
        extraction, and teardown in one call without requiring an output file.

        Args:
            url: Target URL to scrape.
            extract_js: JavaScript expression to extract content. Must return
                        a string. Default: ``"document.body.innerText"``.
            load_wait: Seconds to wait after navigation for JS to render.

        Returns:
            Extracted page text, or an empty string on failure.

        Example::

            scraper = CDPScraper()
            text = scraper.scrape_single_page("https://example.com/article")
            print(text[:500])
        """
        return self.scrape_single(url=url, extract_js=extract_js, load_wait=load_wait)

    def scrape_pages(
        self,
        base_url: str,
        total_pages: int,
        url_pattern: str = "{base}-{page}.html",
        first_page_url: Optional[str] = None,
        extract_js: str = DEFAULT_EXTRACT_JS,
        delay: float = 0.8,
        title_keyword: str = "",
        load_wait: float = 2.0,
    ) -> None:
        """Scrape multiple sequential pages from a (possibly protected) website.

        Launches Chrome, waits for any anti-bot challenge to clear, then
        navigates through each page, extracts content via JavaScript, and
        writes it to numbered text files in ``output_dir``. Pages that already
        have a non-trivial output file are skipped automatically.

        Args:
            base_url: Base URL without a page-number suffix.
            total_pages: Number of pages to scrape.
            url_pattern: Template for page URLs. ``{base}`` is replaced with
                         ``base_url`` and ``{page}`` with the page number.
                         Default: ``"{base}-{page}.html"``.
            first_page_url: Override for the page-1 URL when it differs from
                            the pattern (e.g., no ``-1`` suffix).
            extract_js: JavaScript expression evaluated on each page. Must
                        return a string. Defaults to a forum post extractor.
            delay: Seconds to wait between page navigations (default: 0.8).
                   Set to 0 for maximum speed (may trigger rate limits).
            title_keyword: If set, block until a tab title contains this
                           string before starting — useful for Cloudflare.
            load_wait: Seconds to wait after each navigation for JS to render
                       before extracting content (default: 2.0).
        """
        delay = max(delay, 0.0)

        if not first_page_url:
            first_page_url = (base_url + ".html") if "{page}" in url_pattern else base_url

        domain = _domain_from_url(first_page_url)

        print(f"Launching Chrome -> {first_page_url}")
        self._launch_chrome(first_page_url)

        print("Waiting for anti-bot challenge to clear...")
        tab = self._wait_for_page(keyword=title_keyword)
        if not tab:
            print("ERROR: Could not connect to a Chrome tab.")
            return

        ws_url: str = tab["webSocketDebuggerUrl"]
        print(f"Connected: {tab.get('title', '?')}\n")

        # Save cookie after a successful challenge solve
        self._maybe_save_cookies(ws_url, domain)

        succeeded = 0
        failed = 0
        eta = _ETATracker(total_pages)

        for page_num in range(1, total_pages + 1):
            output_file = os.path.join(self.output_dir, f"page{page_num:03d}.txt")

            # Skip pages that were already scraped successfully
            if os.path.exists(output_file) and os.path.getsize(output_file) > 500:
                if self.show_progress:
                    bar = self._progress_bar(page_num, total_pages, eta.eta_str())
                    print(f"\r{bar}  [SKIP] page {page_num:03d}", end="", flush=True)
                else:
                    print(f"[SKIP] Page {page_num:03d}")
                eta.tick()
                continue

            if page_num == 1:
                url = first_page_url
            else:
                url = url_pattern.replace("{base}", base_url).replace("{page}", str(page_num))

            if not self.show_progress:
                print(f"[{page_num:03d}/{total_pages}] {url}")

            content: Optional[str] = None
            last_error: Optional[Exception] = None

            for attempt in range(1, self.max_retries + 1):
                try:
                    if page_num > 1 or attempt > 1:
                        self._navigate(ws_url, url)
                        time.sleep(load_wait)

                        # Wait out any per-page anti-bot interstitial
                        for _ in range(10):
                            title = self._evaluate_js(ws_url, "document.title") or ""
                            if not any(w in title.lower() for w in _CHALLENGE_WORDS):
                                break
                            time.sleep(2)

                    content = self._evaluate_js(ws_url, extract_js)
                    last_error = None
                    break

                except Exception as exc:
                    last_error = exc
                    if attempt < self.max_retries:
                        backoff = attempt * 2
                        print(
                            f"  Retry {attempt}/{self.max_retries - 1} "
                            f"after error ({exc}) — waiting {backoff}s"
                        )
                        time.sleep(backoff)

            if last_error is not None:
                print(f"  ERROR after {self.max_retries} attempts: {last_error}")
                failed += 1
                eta.tick()
                continue

            header = f"=== PAGE {page_num} OF {total_pages} ===\n\n"
            with open(output_file, "w", encoding="utf-8") as fh:
                fh.write(header + (content or "[empty]"))

            size = os.path.getsize(output_file)
            succeeded += 1
            eta.tick()

            if self.show_progress:
                bar = self._progress_bar(page_num, total_pages, eta.eta_str())
                print(f"\r{bar}  {size:,} bytes    ", end="", flush=True)
            else:
                print(f"  OK — {size:,} bytes")

            time.sleep(delay)

        if self.show_progress:
            print()  # newline after the final progress bar update

        print(f"\nDone. {succeeded} succeeded, {failed} failed -> {self.output_dir}")

    def scrape_single(
        self,
        url: str,
        extract_js: str = "document.body.innerText",
        output_file: Optional[str] = None,
        load_wait: float = 2.0,
    ) -> str:
        """Scrape a single page and return its extracted content.

        Launches Chrome, navigates to the URL, waits for the page to settle,
        evaluates the extraction JS, and optionally writes the result to disk.
        Chrome is terminated automatically after extraction.

        Args:
            url: Target URL to scrape.
            extract_js: JavaScript expression to extract content. Must return
                        a string. Default: ``"document.body.innerText"``.
            output_file: Optional path to write the extracted content to.
            load_wait: Seconds to wait after page load for JS to render.

        Returns:
            Extracted content as a string, or an empty string on failure.
        """
        domain = _domain_from_url(url)

        self._launch_chrome(url)
        tab = self._wait_for_page()
        if not tab:
            return ""

        ws_url: str = tab["webSocketDebuggerUrl"]

        # Try restoring a cached Cloudflare cookie before waiting
        self._maybe_restore_cookies(ws_url, domain)

        time.sleep(load_wait)
        content = self._evaluate_js(ws_url, extract_js) or ""

        # Cache any clearance cookie we received
        self._maybe_save_cookies(ws_url, domain)

        if output_file and content:
            os.makedirs(os.path.dirname(os.path.abspath(output_file)), exist_ok=True)
            with open(output_file, "w", encoding="utf-8") as fh:
                fh.write(content)

        self.close()
        return content

    def close(self) -> None:
        """Terminate the managed Chrome process if it is still running."""
        if self.chrome_proc is not None:
            self.chrome_proc.terminate()
            self.chrome_proc = None


# ── CLI entrypoint ────────────────────────────────────────────────────────────

def main() -> None:
    """Command-line interface for CDP Scraper."""
    parser = argparse.ArgumentParser(
        prog="cdp_scraper",
        description="CDP Scraper — bypass anti-bot protection via real Chrome CDP",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
examples:
  # Scrape a single page
  python cdp_scraper.py --url "https://example.com/page"

  # Scrape 42 pages with custom pattern, 0.5 s delay, and a progress bar
  python cdp_scraper.py --url "https://example.com/thread" --pages 42 \\
      --pattern "{base}-{page}.html" --delay 0.5 --progress

  # Custom JS extraction (e.g. pull only article bodies)
  python cdp_scraper.py --url "https://example.com/thread" --pages 42 \\
      --extract-js "Array.from(document.querySelectorAll('article')).map(e=>e.innerText).join('\\n---\\n')"

  # Headless mode (for sites without anti-bot)
  python cdp_scraper.py --url "https://example.com" --headless

  # Custom Chrome path via environment variable
  CHROME_PATH="/opt/google/chrome/chrome" python cdp_scraper.py --url "..."
        """,
    )
    parser.add_argument("--url", required=True,
                        help="Base URL to scrape")
    parser.add_argument("--pages", type=int, default=1,
                        help="Number of pages to scrape (default: 1)")
    parser.add_argument("--output", default="./output",
                        help="Output directory (default: ./output)")
    parser.add_argument("--delay", type=float, default=0.8,
                        help="Delay between pages in seconds (default: 0.8)")
    parser.add_argument("--pattern", default="{base}-{page}.html",
                        help="URL pattern: {base} and {page} are substituted")
    parser.add_argument("--headless", action="store_true",
                        help="Run Chrome headless (faster, but detectable by some anti-bot systems)")
    parser.add_argument("--progress", action="store_true",
                        help="Show a progress bar with percentage and ETA")
    parser.add_argument("--retries", type=int, default=3,
                        help="Max retry attempts per page on failure (default: 3)")
    parser.add_argument("--port", type=int, default=DEBUG_PORT,
                        help=f"Chrome CDP debug port (default: {DEBUG_PORT})")
    parser.add_argument("--no-cookie-cache", action="store_true",
                        help="Disable Cloudflare cookie caching between runs")
    parser.add_argument(
        "--extract-js",
        metavar="JS",
        default=None,
        help="Custom JavaScript expression to run on each page for content extraction. "
             "Must return a string. Example: \"document.querySelector('main').innerText\"",
    )

    args = parser.parse_args()

    extract_js = args.extract_js if args.extract_js else DEFAULT_EXTRACT_JS

    scraper = CDPScraper(
        output_dir=args.output,
        debug_port=args.port,
        headless=args.headless,
        show_progress=args.progress,
        max_retries=args.retries,
        cache_cookies=not args.no_cookie_cache,
    )

    if args.pages == 1:
        out_file = os.path.join(args.output, "page001.txt")
        content = scraper.scrape_single(args.url, extract_js=extract_js, output_file=out_file)
        print(f"Extracted {len(content):,} characters -> {out_file}")
    else:
        scraper.scrape_pages(
            base_url=args.url,
            total_pages=args.pages,
            url_pattern=args.pattern,
            delay=args.delay,
            extract_js=extract_js,
        )


if __name__ == "__main__":
    main()