diff --git a/.env.example b/.env.example index f892661..be74e3e 100644 --- a/.env.example +++ b/.env.example @@ -16,7 +16,7 @@ BROWSER_ACQUIRE_TIMEOUT_MS=15000 # Recycle a browser after this many temporary challenge contexts. Set 0 to disable. BROWSER_RECYCLE_AFTER_CONTEXTS=8 # Maximum Firefox content processes per browser. -BROWSER_CONTENT_PROCESSES=2 +BROWSER_MAX_CONTENT_PROCESSES=2 # Browsers in the headful sub-pool, used only for DataDome escalations. Warmed on first use # and additional to BROWSER_POOL_SIZE (~380 MB each). Set 1 to scrape DataDome targets. BROWSER_HEADFUL_POOL_SIZE=0 @@ -26,41 +26,45 @@ BROWSER_STALL_TIMEOUT_MS=180000 BROWSER_CLOSE_TIMEOUT_MS=10000 # Maximum time to wait for a browser launch. BROWSER_LAUNCH_TIMEOUT_MS=90000 -# Optional Chrome executable override. -CHROME_EXECUTABLE= - # Optional viewport screenshots (browser tiers only) SCREENSHOT_SETTLE_MS=3000 SCREENSHOT_TIMEOUT_MS=10000 SCREENSHOT_JPEG_QUALITY=60 SCREENSHOT_MAX_BYTES=4000000 -# Optional browser diagnostics (browser tiers only) -CAPTURE_MAX_CONSOLE_ENTRIES=500 -CAPTURE_MAX_NETWORK_ENTRIES=1000 -CAPTURE_MAX_STRING_CHARS=2000 -CAPTURE_MAX_TOTAL_CHARS=1000000 -CAPTURE_MAX_REDIRECT_ENTRIES=50 -CAPTURE_SIZES_TIMEOUT_MS=2000 -CAPTURE_MAX_RESPONSE_PATTERNS=10 +# Optional console and network diagnostics (browser tiers only) +DIAGNOSTICS_MAX_CONSOLE_ENTRIES=500 +DIAGNOSTICS_MAX_NETWORK_ENTRIES=1000 +DIAGNOSTICS_MAX_STRING_CHARS=2000 +DIAGNOSTICS_MAX_TOTAL_CHARS=1000000 +DIAGNOSTICS_SIZE_TIMEOUT_MS=2000 + +# Optional main-document redirect chain (browser tiers only) +REDIRECT_MAX_ENTRIES=50 +REDIRECT_MAX_URL_CHARS=2000 +REDIRECT_MAX_TOTAL_CHARS=1000000 + +# Optional matched response-body capture (browser tiers only) +CAPTURE_MAX_PATTERNS=10 CAPTURE_MAX_RESPONSES=5 -CAPTURE_MAX_RESPONSE_BYTES=5242880 -CAPTURE_MAX_RESPONSE_TOTAL_BYTES=10485760 +CAPTURE_MAX_BODY_BYTES=5242880 +CAPTURE_MAX_TOTAL_BYTES=10485760 CAPTURE_MAX_READ_BYTES=10485760 +CAPTURE_MAX_METADATA_CHARS=2000 CAPTURE_BODY_TIMEOUT_MS=5000 CAPTURE_SETTLE_MS=15000 CAPTURE_MAX_SETTLE_MS=60000 -CAPTURE_SETTLE_IDLE_FLOOR_MS=5000 +CAPTURE_IDLE_FLOOR_MS=5000 # Sessions -# Redis stores solved cookies and browser sessions. +# Redis stores solved cookies and browser sessions. Leave empty to disable the cache. REDIS_URL=redis://localhost:6379 +# Lifetime of Redis-cached sessions in seconds. +REDIS_SESSION_TTL_SECONDS=3600 # Maximum time for each Redis connection attempt. REDIS_CONNECT_TIMEOUT_MS=5000 # Delay between background reconnect attempts. Set 0 to disable retry. REDIS_RETRY_DELAY_MS=5000 -# Lifetime of cached sessions in seconds. -SESSION_TTL_SECONDS=3600 # Upstream proxies # Tier 3 datacenter proxy: HTTP, authenticated HTTP, or SOCKS5. @@ -82,18 +86,18 @@ RESIDENTIAL_PROXY_LIST_FILE= # HTTP/HTTPS challenge-bypassing proxy # Enables the general proxy listener. -MITM_PROXY_ENABLED=false +MITM_ENABLED=false # Proxy address and port. Use 127.0.0.1 for local-only access. -MITM_PROXY_HOST=0.0.0.0 -MITM_PROXY_PORT=8192 +MITM_HOST=0.0.0.0 +MITM_PORT=8192 # Persistent directory for the generated CA certificate and key. -MITM_PROXY_CA_DIR=/data/proxy-ca +MITM_CA_DIR=/data/proxy-ca # Highest allowed escalation tier. Leave empty to allow Tier 4. -MITM_PROXY_MAX_TIER= +MITM_MAX_TIER= # Skip the direct Tier 0 probe and route requests through the scraper. Disables Tier 0 streaming. -MITM_PROXY_ALWAYS_SCRAPE=false +MITM_ALWAYS_SCRAPE=false # Logs proxied requests. Errors are always logged. -MITM_PROXY_DEBUG=false +MITM_DEBUG=false # CAPTCHA audio # Optional Whisper-compatible speech-to-text endpoint. diff --git a/CHANGELOG.md b/CHANGELOG.md index e8a5445..c125815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Reorganized runtime configuration into concise subsystem namespaces. Console/network limits now use `DIAGNOSTICS_*`, redirect-chain limits use `REDIRECT_*`, response-body limits remain under `CAPTURE_*`, forward-proxy settings use `MITM_*`, and Redis session expiry is `REDIS_SESSION_TTL_SECONDS`. `BROWSER_MAX_CONTENT_PROCESSES` now makes the process cap explicit; the unused `CHROME_EXECUTABLE` entry was removed. These names replace the previous environment variables without compatibility aliases; the [configuration migration guide](apps/docs/deployment/configuration-migration.md) contains the complete mapping. +- An empty or unset `REDIS_URL` now disables the optional session cache without attempting a localhost connection or requiring a separate enable flag. +- Supplied Compose variants now explicitly pass every supported runtime tuning variable from `.env` into the container instead of silently ignoring screenshot, diagnostics, redirect, response-capture, and CAPTCHA settings. +- Renamed internal byte/character limits to `STREAM_THRESHOLD_BYTES` and `MCP_HTML_MAX_CHARS`, and namespaced the Camoufox font-retention build argument as `CAMOUFOX_KEEP_SPOOFED_OS_FONTS`. + ### Added -- Optional `MITM_PROXY_ALWAYS_SCRAPE=true` mode skips the forward proxy's direct Tier 0 probe and routes ordinary HTTP requests immediately into the existing scraper ladder for sites where the probe itself triggers a temporary ban (#93). WebSocket relays remain direct; the mode is off by default and documented as unsuitable for general media/download traffic because it bypasses Tier 0 streaming. +- Optional `MITM_ALWAYS_SCRAPE=true` mode skips the forward proxy's direct Tier 0 probe and routes ordinary HTTP requests immediately into the existing scraper ladder for sites where the probe itself triggers a temporary ban (#93). WebSocket relays remain direct; the mode is off by default and documented as unsuitable for general media/download traffic because it bypasses Tier 0 streaming. - **DataDome support.** Detect Device Check, slider CAPTCHA and `t=bv` hard blocks from challenge markers and `x-dd-b`. Device Check uses a dedicated waiter and an optional headful Xvfb pool; the slider is reported as `datadome-captcha-required`. Enable startup-warmed capacity with `BROWSER_HEADFUL_POOL_SIZE=1` (off by default). - **AWS WAF Challenge support.** Detect the documented `202` Challenge and `405` CAPTCHA responses from their `x-amzn-waf-action` header, with a conservative two-marker HTML fallback. Silent challenges use a dedicated browser waiter for the domain-matching `aws-waf-token`; interactive CAPTCHA is surfaced as `aws-waf-captcha-required` for a future solver. - Optional response-body capture: `captureResponses` on `POST /scrape` takes URL patterns (a substring, or a glob when the pattern contains `*` or `?`) and returns the matching responses' bodies in `ScrapeResult.capturedResponses`, so a page that ships an empty shell and loads its content over a background request is still readable. `settleTimeout` holds the page open after load waiting for a match and `waitForSelector` ends that window early. Off by default — without patterns no listener is attached. Pattern count, body count, per-body bytes and total bytes are bounded and tunable via `CAPTURE_*`; a body over its budget comes back trimmed and flagged `truncated`, a binary or unknown content type comes back base64, and a body that cannot be read carries its own `error` rather than failing the scrape. -- Optional console, network and redirect-chain capture: `consoleLogs`, `networkLogs` and `redirectChain` on `POST /scrape` return the page's console messages, per-request resource timings, and the URLs the main document walked (`ScrapeResult.consoleLogs` / `networkLogs` / `redirectChain`), captured by the browser tiers (2-4). Each flag is independent and off by default — without it no listener is attached and nothing is buffered. Entry counts, string lengths and total captured characters are bounded and tunable via `CAPTURE_*`; anything past a cap is dropped whole rather than truncated, and a capture failure leaves the field unset rather than failing the scrape. +- Optional console, network and redirect-chain capture: `consoleLogs`, `networkLogs` and `redirectChain` on `POST /scrape` return the page's console messages, per-request resource timings, and the URLs the main document walked (`ScrapeResult.consoleLogs` / `networkLogs` / `redirectChain`), captured by the browser tiers (2-4). Each flag is independent and off by default — without it no listener is attached and nothing is buffered. Diagnostics use `DIAGNOSTICS_*` limits and redirect chains use independent `REDIRECT_*` limits; anything past a cap is dropped whole rather than truncated, and a capture failure leaves the field unset rather than failing the scrape. - Optional viewport screenshot: `screenshot: true` on `POST /scrape` returns a base64 JPEG of the viewport in `ScrapeResult.screenshot`, captured by the browser tiers (2-4) immediately before the HTML read so image and markup describe the same moment. Off by default; a stock request attaches nothing and does no extra work. Settle wait, capture timeout, JPEG quality, and maximum image size are bounded and tunable via `SCREENSHOT_*`, and a capture failure leaves the field unset rather than failing the scrape. ### Fixed diff --git a/README.md b/README.md index c1e6c00..4b63ade 100644 --- a/README.md +++ b/README.md @@ -143,15 +143,15 @@ cookie isn't portable. For those indexers, enable TRAWL's forward proxy and add Prowlarr as an **HTTP proxy**: ```env -MITM_PROXY_ENABLED=true -MITM_PROXY_PORT=8192 -MITM_PROXY_CA_DIR=/data/proxy-ca # persist the CA (mount a volume) -MITM_PROXY_MAX_TIER=4 # cap escalation (e.g. 3 to stay off residential) -MITM_PROXY_ALWAYS_SCRAPE=false # opt in to bypass the proxy's direct Tier 0 probe +MITM_ENABLED=true +MITM_PORT=8192 +MITM_CA_DIR=/data/proxy-ca # persist the CA (mount a volume) +MITM_MAX_TIER=4 # cap escalation (e.g. 3 to stay off residential) +MITM_ALWAYS_SCRAPE=false # opt in to bypass the proxy's direct Tier 0 probe ``` By default the listener binds `0.0.0.0` so clients on a Docker bridge network can reach -it; set `MITM_PROXY_HOST=127.0.0.1` to restrict it to loopback on a bare-metal host. +it; set `MITM_HOST=127.0.0.1` to restrict it to loopback on a bare-metal host. 1. Install the proxy's CA into the client's trust store so it accepts the per-host certs: `curl http://:8191/proxy-ca.crt` → add to the Prowlarr container's CA store @@ -175,7 +175,7 @@ supported traffic, limitations, CA installation, and client examples. ### Installing the proxy CA certificate The proxy self-generates a root CA on first run. Its certificate and private key are persisted -under `MITM_PROXY_CA_DIR` (default `/data/proxy-ca`). Per-host certificates are minted and cached +under `MITM_CA_DIR` (default `/data/proxy-ca`). Per-host certificates are minted and cached in memory while TRAWL runs; they do not need separate installation because they are signed by the persistent root. Every client that uses the proxy must trust that root. Without it, HTTPS fails with `ERR_CERT_AUTHORITY_INVALID` (browsers) or `PKIX path building failed` (Java). @@ -297,7 +297,7 @@ Get-ChildItem Cert:\LocalMachine\Root | Where-Object { $_.Subject -like "*TRAWL Every installation method has a symmetric removal path. Search your trust store for `TRAWL MITM Proxy CA` (the CA's CN) and delete that entry. The CA certificate and key also live at -`/ca.crt` and `ca.key` on the TRAWL host. Deleting either causes TRAWL to +`/ca.crt` and `ca.key` on the TRAWL host. Deleting either causes TRAWL to generate a new root on its next start, so existing clients must install the new certificate. ## Tiers @@ -393,9 +393,9 @@ for pool and mounted-file examples. | `BROWSER_POOL_SIZE` | `3` | Warm Camoufox Firefox instances | | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000` | How long `acquire()` polls for a free browser before HTTP 429 is returned | | `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8` | Rolling-replace after this many Tier 3/4 contexts; set `0` to disable | -| `BROWSER_CONTENT_PROCESSES` | `2` | Cap Firefox content processes per browser (`dom.ipc.processCount`); lowers RAM/CPU | -| `SESSION_TTL_SECONDS` | `3600` | Redis session cache TTL (seconds) | -| `REDIS_URL` | `redis://localhost:6379` | Redis connection string | +| `BROWSER_MAX_CONTENT_PROCESSES` | `2` | Cap Firefox content processes per browser (`dom.ipc.processCount`); lowers RAM/CPU | +| `REDIS_SESSION_TTL_SECONDS` | `3600` | Redis session cache TTL (seconds) | +| `REDIS_URL` | — | Redis connection string; empty or unset disables the session cache | | `REDIS_CONNECT_TIMEOUT_MS` | `5000` | Maximum time for each Redis connection attempt | | `REDIS_RETRY_DELAY_MS` | `5000` | Delay before reconnecting after startup failure; `0` disables retry | | `PROXY_URL` | — | Optional Tier 3 HTTP or SOCKS5 proxy, or comma-separated pool | @@ -404,13 +404,17 @@ for pool and mounted-file examples. | `RESIDENTIAL_PROXY_LIST_FILE` | — | File containing one Tier 4 proxy URL per line | | `STT_URL` | — | Local Whisper endpoint for reCAPTCHA (optional) | | `PORT` | `8191` | API listen port | -| `MITM_PROXY_ENABLED` | `false` | Enable the challenge-bypassing HTTP/HTTPS proxy | -| `MITM_PROXY_PORT` | `8192` | Forward-proxy listen port | -| `MITM_PROXY_HOST` | `0.0.0.0` | Bind address; `127.0.0.1` for loopback-only | -| `MITM_PROXY_CA_DIR` | `/data/proxy-ca` | Persistent root CA certificate and private-key directory | -| `MITM_PROXY_MAX_TIER` | `4` | Cap escalation used by the proxy (e.g. `3` to stay off residential) | -| `MITM_PROXY_ALWAYS_SCRAPE` | `false` | Skip proxy Tier 0; disables the direct media/large-file streaming path | -| `MITM_PROXY_DEBUG` | `false` | Log one line per proxied request (errors are always logged) | +| `MITM_ENABLED` | `false` | Enable the challenge-bypassing HTTP/HTTPS proxy | +| `MITM_PORT` | `8192` | Forward-proxy listen port | +| `MITM_HOST` | `0.0.0.0` | Bind address; `127.0.0.1` for loopback-only | +| `MITM_CA_DIR` | `/data/proxy-ca` | Persistent root CA certificate and private-key directory | +| `MITM_MAX_TIER` | `4` | Cap escalation used by the proxy (e.g. `3` to stay off residential) | +| `MITM_ALWAYS_SCRAPE` | `false` | Skip proxy Tier 0; disables the direct media/large-file streaming path | +| `MITM_DEBUG` | `false` | Log one line per proxied request (errors are always logged) | + +Upgrading from an earlier release requires renaming several environment variables. See the +[configuration migration guide](apps/docs/deployment/configuration-migration.md) for the complete +old-to-new mapping and Redis opt-in behavior. ## Stack diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index eaa371e..b09e6be 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -57,8 +57,8 @@ ARG TARGETARCH # MacIntel -> "macos"), and Camoufox spoofs that OS's font list with it. Dropping those # bundles therefore leaves a browser advertising fonts whose files are gone: every glyph # renders as a tofu box, and the fingerprint is self-inconsistent for anti-bot scoring. -# Set KEEP_SPOOFED_OS_FONTS=1 to keep them (+891MB) when rendered output matters. -ARG KEEP_SPOOFED_OS_FONTS=0 +# Set CAMOUFOX_KEEP_SPOOFED_OS_FONTS=1 to keep them (+891MB) when rendered output matters. +ARG CAMOUFOX_KEEP_SPOOFED_OS_FONTS=0 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \ case "$TARGETARCH" in \ amd64) ZIP="camoufox-152.0.4-beta.28-lin.x86_64.zip"; SHA256="924f3109ccd6d47cd6a0384d67a345fadf975d48b6319f8dbbd5954c588982bd" ;; \ @@ -73,7 +73,7 @@ RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \ rm /tmp/camoufox.zip && \ printf '{"version":"152.0.4","release":"beta.28"}\n' > /opt/camoufox/version.json && \ chmod -R 755 /opt/camoufox && \ - if [ "$KEEP_SPOOFED_OS_FONTS" != "1" ]; then \ + if [ "$CAMOUFOX_KEEP_SPOOFED_OS_FONTS" != "1" ]; then \ rm -rf /opt/camoufox/fonts/macos /opt/camoufox/fonts/windows; \ fi diff --git a/apps/api/Dockerfile.baseline b/apps/api/Dockerfile.baseline index 0645aa6..033d3df 100644 --- a/apps/api/Dockerfile.baseline +++ b/apps/api/Dockerfile.baseline @@ -57,8 +57,8 @@ ARG TARGETARCH # MacIntel -> "macos"), and Camoufox spoofs that OS's font list with it. Dropping those # bundles therefore leaves a browser advertising fonts whose files are gone: every glyph # renders as a tofu box, and the fingerprint is self-inconsistent for anti-bot scoring. -# Set KEEP_SPOOFED_OS_FONTS=1 to keep them (+891MB) when rendered output matters. -ARG KEEP_SPOOFED_OS_FONTS=0 +# Set CAMOUFOX_KEEP_SPOOFED_OS_FONTS=1 to keep them (+891MB) when rendered output matters. +ARG CAMOUFOX_KEEP_SPOOFED_OS_FONTS=0 RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \ case "$TARGETARCH" in \ amd64) ZIP="camoufox-152.0.4-beta.28-lin.x86_64.zip"; SHA256="924f3109ccd6d47cd6a0384d67a345fadf975d48b6319f8dbbd5954c588982bd" ;; \ @@ -73,7 +73,7 @@ RUN --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \ rm /tmp/camoufox.zip && \ printf '{"version":"152.0.4","release":"beta.28"}\n' > /opt/camoufox/version.json && \ chmod -R 755 /opt/camoufox && \ - if [ "$KEEP_SPOOFED_OS_FONTS" != "1" ]; then \ + if [ "$CAMOUFOX_KEEP_SPOOFED_OS_FONTS" != "1" ]; then \ rm -rf /opt/camoufox/fonts/macos /opt/camoufox/fonts/windows; \ fi diff --git a/apps/api/src/config.test.ts b/apps/api/src/config.test.ts new file mode 100644 index 0000000..968edfd --- /dev/null +++ b/apps/api/src/config.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from "bun:test" + +type ConfigSnapshot = { + redisUrl: string | null + redisSessionTtlSeconds: number + poolSize: number + maxContentProcesses: number + acquireTimeoutMs: number + recycleAfterContexts: number + headfulPoolSize: number + stallTimeoutMs: number + closeTimeoutMs: number + launchTimeoutMs: number + port: number + mitmPort: number +} + +const readConfig = (overrides: Record): ConfigSnapshot => { + const script = ` + const config = await import("./config.ts") + console.log(JSON.stringify({ + redisUrl: config.REDIS_URL ?? null, + redisSessionTtlSeconds: config.REDIS_SESSION_TTL_SECONDS, + poolSize: config.POOL_SIZE, + maxContentProcesses: config.BROWSER_MAX_CONTENT_PROCESSES, + acquireTimeoutMs: config.ACQUIRE_TIMEOUT_MS, + recycleAfterContexts: config.RECYCLE_AFTER_TEMPORARY_CONTEXTS, + headfulPoolSize: config.HEADFUL_POOL_SIZE, + stallTimeoutMs: config.STALL_TIMEOUT_MS, + closeTimeoutMs: config.CLOSE_TIMEOUT_MS, + launchTimeoutMs: config.LAUNCH_TIMEOUT_MS, + port: config.PORT, + mitmPort: config.MITM_PORT, + })) + ` + const result = Bun.spawnSync({ + cmd: [process.execPath, "-e", script], + cwd: import.meta.dir, + env: { ...process.env, ...overrides }, + }) + expect(result.exitCode).toBe(0) + return JSON.parse(result.stdout.toString()) as ConfigSnapshot +} + +describe("environment configuration", () => { + test("reads the renamed variables and trims REDIS_URL", () => { + expect( + readConfig({ + REDIS_URL: " redis://cache.test:6379/2 ", + REDIS_SESSION_TTL_SECONDS: "7200", + BROWSER_POOL_SIZE: "4", + BROWSER_MAX_CONTENT_PROCESSES: "3", + BROWSER_ACQUIRE_TIMEOUT_MS: "12000", + BROWSER_RECYCLE_AFTER_CONTEXTS: "0", + BROWSER_HEADFUL_POOL_SIZE: "2", + BROWSER_STALL_TIMEOUT_MS: "90000", + BROWSER_CLOSE_TIMEOUT_MS: "8000", + BROWSER_LAUNCH_TIMEOUT_MS: "45000", + PORT: "9000", + MITM_PORT: "9001", + }), + ).toEqual({ + redisUrl: "redis://cache.test:6379/2", + redisSessionTtlSeconds: 7200, + poolSize: 4, + maxContentProcesses: 3, + acquireTimeoutMs: 12000, + recycleAfterContexts: 0, + headfulPoolSize: 2, + stallTimeoutMs: 90000, + closeTimeoutMs: 8000, + launchTimeoutMs: 45000, + port: 9000, + mitmPort: 9001, + }) + }) + + test("disables Redis for a blank URL and safely rejects malformed numeric values", () => { + expect( + readConfig({ + REDIS_URL: " ", + REDIS_SESSION_TTL_SECONDS: "-1", + SESSION_TTL_SECONDS: "99", + BROWSER_POOL_SIZE: "NaN", + BROWSER_MAX_CONTENT_PROCESSES: "0", + BROWSER_CONTENT_PROCESSES: "99", + BROWSER_ACQUIRE_TIMEOUT_MS: "-5", + BROWSER_RECYCLE_AFTER_CONTEXTS: "-1", + BROWSER_HEADFUL_POOL_SIZE: "1.5", + BROWSER_STALL_TIMEOUT_MS: "Infinity", + BROWSER_CLOSE_TIMEOUT_MS: "0", + BROWSER_LAUNCH_TIMEOUT_MS: "unsafe", + PORT: "70000", + MITM_PORT: "0", + }), + ).toEqual({ + redisUrl: null, + redisSessionTtlSeconds: 3600, + poolSize: 3, + maxContentProcesses: 2, + acquireTimeoutMs: 15000, + recycleAfterContexts: 8, + headfulPoolSize: 0, + stallTimeoutMs: 180000, + closeTimeoutMs: 10000, + launchTimeoutMs: 90000, + port: 8191, + mitmPort: 8192, + }) + }) +}) diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index 180d26f..2eca2ca 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -1,38 +1,38 @@ import { ProxyPool } from "@trawl/tiers" -export const REDIS_URL = process.env.REDIS_URL ?? "redis://localhost:6379" -export const PORT = Number(process.env.PORT ?? "8191") -export const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3") +export const REDIS_URL = process.env.REDIS_URL?.trim() || undefined +const integerInRange = (value: string | undefined, fallback: number, min: number, max = Number.MAX_SAFE_INTEGER) => { + if (value === undefined || value.trim() === "") return fallback + const parsed = Number(value) + return Number.isSafeInteger(parsed) && parsed >= min && parsed <= max ? parsed : fallback +} +const positiveInteger = (value: string | undefined, fallback: number): number => integerInRange(value, fallback, 1) +const nonNegativeInteger = (value: string | undefined, fallback: number): number => integerInRange(value, fallback, 0) + +export const PORT = integerInRange(process.env.PORT, 8_191, 1, 65_535) +export const POOL_SIZE = positiveInteger(process.env.BROWSER_POOL_SIZE, 3) // How long acquire() will poll for a free browser before rejecting with PoolExhaustedError. // 15s covers a full CF challenge burst with pool=3 (queue depth 7, slowest finishes at ~12s). // Tune lower for fast-fail feedback in dev; tune higher for very heavy upstream targets. -export const ACQUIRE_TIMEOUT_MS = Number(process.env.BROWSER_ACQUIRE_TIMEOUT_MS ?? "15000") -export const SESSION_TTL = Number(process.env.SESSION_TTL_SECONDS ?? "3600") -const positiveInteger = (value: string | undefined, fallback: number): number => { - const parsed = Number(value) - return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback -} -const nonNegativeInteger = (value: string | undefined, fallback: number): number => { - const parsed = Number(value) - return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : fallback -} +export const ACQUIRE_TIMEOUT_MS = positiveInteger(process.env.BROWSER_ACQUIRE_TIMEOUT_MS, 15_000) +export const REDIS_SESSION_TTL_SECONDS = positiveInteger(process.env.REDIS_SESSION_TTL_SECONDS, 3_600) // A failed initial Redis connection must not disable Tier 2 for the process lifetime. // Each attempt is bounded; failed attempts are retried in the background while the API stays ready. export const REDIS_CONNECT_TIMEOUT_MS = positiveInteger(process.env.REDIS_CONNECT_TIMEOUT_MS, 5_000) export const REDIS_RETRY_DELAY_MS = nonNegativeInteger(process.env.REDIS_RETRY_DELAY_MS, 5_000) // Rolling-replace a browser after this many Tier 3/4 temporary contexts. Every // creation counts regardless of outcome; 0 disables periodic replacement. -export const RECYCLE_AFTER_TEMPORARY_CONTEXTS = Number(process.env.BROWSER_RECYCLE_AFTER_CONTEXTS ?? "8") +export const RECYCLE_AFTER_TEMPORARY_CONTEXTS = nonNegativeInteger(process.env.BROWSER_RECYCLE_AFTER_CONTEXTS, 8) // Caps Firefox content processes per browser. Default `2` keeps thread/RAM footprint // minimal while still allowing CF/Imperva challenges to resolve. Raise if specific // targets fail with empty content (rare). -export const CONTENT_PROCESSES = Number(process.env.BROWSER_CONTENT_PROCESSES ?? "2") +export const BROWSER_MAX_CONTENT_PROCESSES = positiveInteger(process.env.BROWSER_MAX_CONTENT_PROCESSES, 2) // Size of the headful sub-pool, launched behind Xvfb for DataDome Device Check escalations. // // Off by default because this pool sits ON TOP of BROWSER_POOL_SIZE: one headful browser // plus its X display measures ~380 MB, which would silently move the memory ceiling of a // deployment that never meets DataDome. Set it to 1 to scrape DataDome targets. -export const HEADFUL_POOL_SIZE = Number(process.env.BROWSER_HEADFUL_POOL_SIZE ?? "0") +export const HEADFUL_POOL_SIZE = nonNegativeInteger(process.env.BROWSER_HEADFUL_POOL_SIZE, 0) // Optional MCP Streamable HTTP endpoint. Keep this disabled unless the API is // reachable only by trusted clients; v1 intentionally has no authentication. @@ -45,13 +45,13 @@ export const MCP_ALLOWED_ORIGINS = (process.env.MCP_ALLOWED_ORIGINS ?? "") // How long a browser may stay checked out before the pool calls it wedged rather than // busy. A scrape's own budget is req.maxTimeout (default 60s), so 3x that is well clear // of anything legitimate while still catching a hung checkout within a few minutes. -export const STALL_TIMEOUT_MS = Number(process.env.BROWSER_STALL_TIMEOUT_MS ?? "180000") +export const STALL_TIMEOUT_MS = positiveInteger(process.env.BROWSER_STALL_TIMEOUT_MS, 180_000) // Upper bound on a single browser/context close during a recycle. Camoufox can hang on // close when a content process is wedged; past this we abandon the close and relaunch. -export const CLOSE_TIMEOUT_MS = Number(process.env.BROWSER_CLOSE_TIMEOUT_MS ?? "10000") +export const CLOSE_TIMEOUT_MS = positiveInteger(process.env.BROWSER_CLOSE_TIMEOUT_MS, 10_000) // Upper bound on a browser launch. A cold Camoufox start is a few seconds, but launches // have been observed to hang indefinitely — without a bound that strands the pool entry. -export const LAUNCH_TIMEOUT_MS = Number(process.env.BROWSER_LAUNCH_TIMEOUT_MS ?? "90000") +export const LAUNCH_TIMEOUT_MS = positiveInteger(process.env.BROWSER_LAUNCH_TIMEOUT_MS, 90_000) // PROXY_URL / RESIDENTIAL_PROXY_URL accept a comma-separated list of proxy URLs (a single // URL still works — it's just a 1-element list). *_LIST_FILE is an alternative source @@ -64,28 +64,28 @@ export const residentialProxyPool = ProxyPool.fromEnv( // ── MITM forward-proxy mode ──────────────────────────────────────────────────── // Optional browser-backed HTTP(S) forward proxy (apps/api/src/proxy). Off by default. -// When enabled, point a client's HTTP(S) proxy at MITM_PROXY_PORT and every request is +// When enabled, point a client's HTTP(S) proxy at MITM_PORT and every request is // re-issued through the browser pool — for clients that only consume cookies+UA from // /v1 and re-fetch themselves, which fails on fingerprint-bound Cloudflare clearances. // See proxy/server.ts for the full rationale. -export const MITM_PROXY_ENABLED = /^(1|true|yes)$/i.test(process.env.MITM_PROXY_ENABLED ?? "") -export const MITM_PROXY_PORT = Number(process.env.MITM_PROXY_PORT ?? "8192") +export const MITM_ENABLED = /^(1|true|yes)$/i.test(process.env.MITM_ENABLED ?? "") +export const MITM_PORT = integerInRange(process.env.MITM_PORT, 8_192, 1, 65_535) // Default 0.0.0.0 — the dominant deployment is docker-compose (clients reach trawl // through the docker bridge, which requires a non-loopback bind). Loopback-only -// operators can set MITM_PROXY_HOST=127.0.0.1. The primary safety guard remains -// MITM_PROXY_ENABLED=false. -export const MITM_PROXY_HOST = process.env.MITM_PROXY_HOST ?? "0.0.0.0" +// operators can set MITM_HOST=127.0.0.1. The primary safety guard remains +// MITM_ENABLED=false. +export const MITM_HOST = process.env.MITM_HOST ?? "0.0.0.0" // CA cert + key live here (persist across restarts so the CA is installed once). -export const MITM_PROXY_CA_DIR = process.env.MITM_PROXY_CA_DIR ?? "/data/proxy-ca" +export const MITM_CA_DIR = process.env.MITM_CA_DIR ?? "/data/proxy-ca" // Cap the tier the proxy will escalate to (e.g. keep it off residential Tier 4). -const configuredMaxTier = Number(process.env.MITM_PROXY_MAX_TIER) +const configuredMaxTier = Number(process.env.MITM_MAX_TIER) const isTier = (tier: number): tier is 1 | 2 | 3 | 4 => tier === 1 || tier === 2 || tier === 3 || tier === 4 -export const MITM_PROXY_MAX_TIER = isTier(configuredMaxTier) ? configuredMaxTier : undefined +export const MITM_MAX_TIER = isTier(configuredMaxTier) ? configuredMaxTier : undefined // Skip the proxy's direct Tier 0 probe and route ordinary HTTP requests into scrape(). // This is separate from ScrapeRequest.skipHttp, which controls scraper Tier 1. -export const MITM_PROXY_ALWAYS_SCRAPE = /^(1|true|yes)$/i.test(process.env.MITM_PROXY_ALWAYS_SCRAPE ?? "") +export const MITM_ALWAYS_SCRAPE = /^(1|true|yes)$/i.test(process.env.MITM_ALWAYS_SCRAPE ?? "") // Log one line per proxied request (method, url, status, content-type, bytes). Off by // default — proxied clients can be chatty. Errors are always logged. -export const MITM_PROXY_DEBUG = /^(1|true|yes)$/i.test(process.env.MITM_PROXY_DEBUG ?? "") +export const MITM_DEBUG = /^(1|true|yes)$/i.test(process.env.MITM_DEBUG ?? "") export const startTime = Date.now() diff --git a/apps/api/src/deps.ts b/apps/api/src/deps.ts index a1e2ed0..77b12f5 100644 --- a/apps/api/src/deps.ts +++ b/apps/api/src/deps.ts @@ -3,8 +3,8 @@ import type { AcquireOptions, OrchestratorDeps } from "@trawl/tiers" import type { SessionData } from "@trawl/types" import { ACQUIRE_TIMEOUT_MS, + BROWSER_MAX_CONTENT_PROCESSES, CLOSE_TIMEOUT_MS, - CONTENT_PROCESSES, HEADFUL_POOL_SIZE, LAUNCH_TIMEOUT_MS, POOL_SIZE, @@ -12,9 +12,9 @@ import { RECYCLE_AFTER_TEMPORARY_CONTEXTS, REDIS_CONNECT_TIMEOUT_MS, REDIS_RETRY_DELAY_MS, + REDIS_SESSION_TTL_SECONDS, REDIS_URL, residentialProxyPool, - SESSION_TTL, STALL_TIMEOUT_MS, } from "./config" @@ -93,16 +93,22 @@ export class SessionCacheRecovery { } } -const sessionCacheRecovery = new SessionCacheRecovery({ - createCache: () => new SessionCache({ redisUrl: REDIS_URL, ttlSeconds: SESSION_TTL }), - connectTimeoutMs: REDIS_CONNECT_TIMEOUT_MS, - retryDelayMs: REDIS_RETRY_DELAY_MS, - onConnected: () => console.log("[api] session cache connected (Tier 2 fast-path enabled)"), - onUnavailable: (err) => { - const retry = REDIS_RETRY_DELAY_MS > 0 ? `; retrying in ${REDIS_RETRY_DELAY_MS}ms` : "" - console.warn(`[api] session cache unavailable — Tier 2 disabled${retry}:`, err instanceof Error ? err.message : err) - }, -}) +const redisUrl = REDIS_URL +const sessionCacheRecovery = redisUrl + ? new SessionCacheRecovery({ + createCache: () => new SessionCache({ redisUrl, ttlSeconds: REDIS_SESSION_TTL_SECONDS }), + connectTimeoutMs: REDIS_CONNECT_TIMEOUT_MS, + retryDelayMs: REDIS_RETRY_DELAY_MS, + onConnected: () => console.log("[api] session cache connected (Tier 2 fast-path enabled)"), + onUnavailable: (err) => { + const retry = REDIS_RETRY_DELAY_MS > 0 ? `; retrying in ${REDIS_RETRY_DELAY_MS}ms` : "" + console.warn( + `[api] session cache unavailable — Tier 2 disabled${retry}:`, + err instanceof Error ? err.message : err, + ) + }, + }) + : undefined interface InitPoolOptions { poolSize?: number @@ -114,10 +120,10 @@ interface InitPoolOptions { export const getPool = () => state.pool export const getHeadfulPool = () => state.headfulPool -const initSessionCache = (): Promise => sessionCacheRecovery.start() +const initSessionCache = (): Promise => sessionCacheRecovery?.start() ?? Promise.resolve() export const shutdownPools = async (): Promise => { - await Promise.all([sessionCacheRecovery.stop(), state.pool?.shutdown(), state.headfulPool?.shutdown()]) + await Promise.all([sessionCacheRecovery?.stop(), state.pool?.shutdown(), state.headfulPool?.shutdown()]) } export const initPool = async ({ @@ -130,7 +136,7 @@ export const initPool = async ({ poolSize, acquireTimeoutMs: ACQUIRE_TIMEOUT_MS, recycleAfterTemporaryContexts: RECYCLE_AFTER_TEMPORARY_CONTEXTS, - contentProcesses: CONTENT_PROCESSES, + contentProcesses: BROWSER_MAX_CONTENT_PROCESSES, stallAfterMs: STALL_TIMEOUT_MS, closeTimeoutMs: CLOSE_TIMEOUT_MS, launchTimeoutMs: LAUNCH_TIMEOUT_MS, @@ -142,7 +148,7 @@ export const initPool = async ({ poolSize: headfulPoolSize, acquireTimeoutMs: ACQUIRE_TIMEOUT_MS, recycleAfterTemporaryContexts: RECYCLE_AFTER_TEMPORARY_CONTEXTS, - contentProcesses: CONTENT_PROCESSES, + contentProcesses: BROWSER_MAX_CONTENT_PROCESSES, virtualDisplay: true, label: "pool:headful", stallAfterMs: STALL_TIMEOUT_MS, @@ -192,17 +198,17 @@ export const getDeps = (): OrchestratorDeps => { }, loadSession: (d: string) => sessionCacheRecovery - .current() + ?.current() ?.load(d) .catch(() => undefined) ?? Promise.resolve(undefined), saveSession: (d: string, data: SessionData) => sessionCacheRecovery - .current() + ?.current() ?.save(d, data) .catch(() => {}) ?? Promise.resolve(), invalidateSession: (d: string) => sessionCacheRecovery - .current() + ?.current() ?.invalidate(d) .catch(() => {}) ?? Promise.resolve(), proxyPool, diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 1009e90..d49a788 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,12 +1,12 @@ import { createApiApp } from "./app" import { - MITM_PROXY_ALWAYS_SCRAPE, - MITM_PROXY_CA_DIR, - MITM_PROXY_DEBUG, - MITM_PROXY_ENABLED, - MITM_PROXY_HOST, - MITM_PROXY_MAX_TIER, - MITM_PROXY_PORT, + MITM_ALWAYS_SCRAPE, + MITM_CA_DIR, + MITM_DEBUG, + MITM_ENABLED, + MITM_HOST, + MITM_MAX_TIER, + MITM_PORT, POOL_SIZE, PORT, } from "./config" @@ -24,15 +24,15 @@ const poolReady = initPool() // Tier 0 does not need a browser, and browser-backed requests already have a // bounded acquire queue. Start accepting proxy traffic while the pool warms. -if (MITM_PROXY_ENABLED) { +if (MITM_ENABLED) { state.proxyHandle = startMitmProxy({ - port: MITM_PROXY_PORT, - host: MITM_PROXY_HOST, - caDir: MITM_PROXY_CA_DIR, + port: MITM_PORT, + host: MITM_HOST, + caDir: MITM_CA_DIR, deps: getDeps(), - maxTier: MITM_PROXY_MAX_TIER, - alwaysScrape: MITM_PROXY_ALWAYS_SCRAPE, - debug: MITM_PROXY_DEBUG, + maxTier: MITM_MAX_TIER, + alwaysScrape: MITM_ALWAYS_SCRAPE, + debug: MITM_DEBUG, }) } diff --git a/apps/api/src/proxy/__tests__/streaming.test.ts b/apps/api/src/proxy/__tests__/streaming.test.ts index ae343b0..a0541b0 100644 --- a/apps/api/src/proxy/__tests__/streaming.test.ts +++ b/apps/api/src/proxy/__tests__/streaming.test.ts @@ -1,14 +1,16 @@ import { describe, expect, test } from "bun:test" -import { STREAM_THRESHOLD, shouldStream } from "../streaming" +import { STREAM_THRESHOLD_BYTES, shouldStream } from "../streaming" describe("shouldStream", () => { describe("size threshold", () => { test("streams responses >= 8 MiB", () => { - expect(shouldStream("https://example.com/file.bin", STREAM_THRESHOLD, "application/octet-stream")).toMatchObject({ + expect( + shouldStream("https://example.com/file.bin", STREAM_THRESHOLD_BYTES, "application/octet-stream"), + ).toMatchObject({ stream: true, reason: "size-threshold", }) - expect(shouldStream("https://example.com/file.bin", STREAM_THRESHOLD + 1, "text/plain")).toMatchObject({ + expect(shouldStream("https://example.com/file.bin", STREAM_THRESHOLD_BYTES + 1, "text/plain")).toMatchObject({ stream: true, reason: "size-threshold", }) @@ -92,7 +94,7 @@ describe("shouldStream", () => { test("empty content-type with explicit length → size decides", () => { expect(shouldStream("https://example.com/", 100, "").stream).toBe(false) - expect(shouldStream("https://example.com/", STREAM_THRESHOLD + 1, "").stream).toBe(true) + expect(shouldStream("https://example.com/", STREAM_THRESHOLD_BYTES + 1, "").stream).toBe(true) }) test("content-type with charset parameter is parsed correctly", () => { diff --git a/apps/api/src/proxy/streaming.ts b/apps/api/src/proxy/streaming.ts index ec45b10..eb16c37 100644 --- a/apps/api/src/proxy/streaming.ts +++ b/apps/api/src/proxy/streaming.ts @@ -8,12 +8,12 @@ // // Decision precedence: // 1. Unknown Content-Length + binary-looking Content-Type → stream (safer). -// 2. Content-Length >= STREAM_THRESHOLD → stream. +// 2. Content-Length >= STREAM_THRESHOLD_BYTES → stream. // 3. Content-Type starts with video/ or audio/ → stream. // 4. URL extension matches a known binary container → stream. // 5. Otherwise → buffer. -export const STREAM_THRESHOLD = 8 * 1024 * 1024 // 8 MiB +export const STREAM_THRESHOLD_BYTES = 8 * 1024 * 1024 // 8 MiB // URL extensions that are almost always large binaries worth streaming. // Audio/video and HLS manifests are streamed even below the size threshold @@ -55,7 +55,7 @@ export function shouldStream(url: string, contentLength?: number, contentType?: } // 2. Size threshold. - if (typeof contentLength === "number" && contentLength >= STREAM_THRESHOLD) { + if (typeof contentLength === "number" && contentLength >= STREAM_THRESHOLD_BYTES) { return { stream: true, reason: "size-threshold" } } diff --git a/apps/api/src/routes/mcp.test.ts b/apps/api/src/routes/mcp.test.ts index bbd4d67..6736b74 100644 --- a/apps/api/src/routes/mcp.test.ts +++ b/apps/api/src/routes/mcp.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test" import { PoolExhaustedError } from "@trawl/browser" import { ScrapeError } from "@trawl/tiers" import type { ScrapeResult } from "@trawl/types" -import { MCP_HTML_LIMIT, mcpRoute } from "./mcp" +import { MCP_HTML_MAX_CHARS, mcpRoute } from "./mcp" const baseResult: ScrapeResult = { url: "https://1.1.1.1/final", @@ -60,7 +60,7 @@ describe("MCP route", () => { poolReady: () => true, runScrape: async (input) => { received = input - return { ...baseResult, html: `secret-cookie-${"x".repeat(MCP_HTML_LIMIT)}` } + return { ...baseResult, html: `secret-cookie-${"x".repeat(MCP_HTML_MAX_CHARS)}` } }, }) const response = await app.handle( @@ -76,7 +76,7 @@ describe("MCP route", () => { totalMs: 12, truncated: true, }) - expect(body.result.content[0].text.length).toBe(MCP_HTML_LIMIT) + expect(body.result.content[0].text.length).toBe(MCP_HTML_MAX_CHARS) expect(JSON.stringify(body)).not.toContain('cookie"') expect(JSON.stringify(body)).not.toContain("secret-agent") }) diff --git a/apps/api/src/routes/mcp.ts b/apps/api/src/routes/mcp.ts index 8e74841..8801e5f 100644 --- a/apps/api/src/routes/mcp.ts +++ b/apps/api/src/routes/mcp.ts @@ -10,7 +10,7 @@ import { MCP_ALLOWED_ORIGINS } from "../config" import { getDeps, getPool } from "../deps" import { assertPublicHttpUrl, createPublicUrlValidator } from "../outbound-policy" -export const MCP_HTML_LIMIT = 50_000 +export const MCP_HTML_MAX_CHARS = 50_000 type RunScrape = (input: { url: string @@ -63,8 +63,8 @@ function createServer(poolReady: () => boolean, runScrape: RunScrape): McpServer // A redirect can reveal a policy violation even if the initial URL was public. // Reject it from the model output as a second line of defense. await assertPublicHttpUrl(result.url) - const truncated = result.html.length > MCP_HTML_LIMIT - const html = result.html.slice(0, MCP_HTML_LIMIT) + const truncated = result.html.length > MCP_HTML_MAX_CHARS + const html = result.html.slice(0, MCP_HTML_MAX_CHARS) const structuredContent = { url: result.url, statusCode: result.statusCode, diff --git a/apps/api/src/routes/proxy-ca.ts b/apps/api/src/routes/proxy-ca.ts index 64914a1..98dbe59 100644 --- a/apps/api/src/routes/proxy-ca.ts +++ b/apps/api/src/routes/proxy-ca.ts @@ -1,16 +1,16 @@ import { existsSync, readFileSync } from "node:fs" import { join } from "node:path" import { Elysia } from "elysia" -import { MITM_PROXY_CA_DIR, MITM_PROXY_ENABLED } from "../config" +import { MITM_CA_DIR, MITM_ENABLED } from "../config" // Serves the MITM proxy CA certificate for easy installation into a client's trust // store (e.g. `curl http://trawl:8191/proxy-ca.crt`). Only mounted when proxy mode is on. export function proxyCaRoute() { const app = new Elysia() - if (!MITM_PROXY_ENABLED) return app + if (!MITM_ENABLED) return app return app.get("/proxy-ca.crt", ({ set }) => { - const path = join(MITM_PROXY_CA_DIR, "ca.crt") + const path = join(MITM_CA_DIR, "ca.crt") if (!existsSync(path)) { set.status = 503 return "CA not generated yet — start the proxy first" diff --git a/apps/docs/.vitepress/config.ts b/apps/docs/.vitepress/config.ts index 6e59d49..75c410f 100644 --- a/apps/docs/.vitepress/config.ts +++ b/apps/docs/.vitepress/config.ts @@ -87,6 +87,7 @@ export default defineConfig({ items: [ { text: "Docker Compose", link: "/deployment/docker-compose" }, { text: "Standalone Containers", link: "/deployment/standalone" }, + { text: "Configuration Migration", link: "/deployment/configuration-migration" }, { text: "Troubleshooting", link: "/deployment/troubleshooting" }, ], }, diff --git a/apps/docs/api-reference/native-api.md b/apps/docs/api-reference/native-api.md index ede976b..b53701f 100644 --- a/apps/docs/api-reference/native-api.md +++ b/apps/docs/api-reference/native-api.md @@ -96,7 +96,7 @@ interface CapturedResponseEntry { headers: Record body: string | null // text, or base64 when binary/unknown; null when unreadable base64Encoded: boolean - truncated: boolean // body trimmed to CAPTURE_MAX_RESPONSE_BYTES + truncated: boolean // body trimmed to CAPTURE_MAX_BODY_BYTES error?: string // why the body is null (read failed, budget spent, ...) } diff --git a/apps/docs/api-reference/overview.md b/apps/docs/api-reference/overview.md index 3864d83..16bf266 100644 --- a/apps/docs/api-reference/overview.md +++ b/apps/docs/api-reference/overview.md @@ -36,8 +36,8 @@ Content-Type: application/json ## Forward proxy -When `MITM_PROXY_ENABLED=true`, TRAWL also listens as an HTTP/HTTPS forward proxy on -`MITM_PROXY_PORT` (default `8192`). This is a socket-level proxy interface rather than a JSON API +When `MITM_ENABLED=true`, TRAWL also listens as an HTTP/HTTPS forward proxy on +`MITM_PORT` (default `8192`). This is a socket-level proxy interface rather than a JSON API endpoint. It forwards normal traffic directly and escalates recognized challenge walls through the same tier engine as `/scrape`. diff --git a/apps/docs/architecture/browser-pool.md b/apps/docs/architecture/browser-pool.md index 73048c7..163503d 100644 --- a/apps/docs/architecture/browser-pool.md +++ b/apps/docs/architecture/browser-pool.md @@ -45,7 +45,7 @@ new BrowserPool({ acquireTimeoutMs: 15000, // BROWSER_ACQUIRE_TIMEOUT_MS — 15s default pollIntervalMs: 100, // how often to re-check for an idle browser recycleAfterTemporaryContexts: 8, - contentProcesses: 2, // BROWSER_CONTENT_PROCESSES — caps Firefox content procs + contentProcesses: 2, // BROWSER_MAX_CONTENT_PROCESSES — caps Firefox content procs stallAfterMs: 180000, // BROWSER_STALL_TIMEOUT_MS closeTimeoutMs: 10000, // BROWSER_CLOSE_TIMEOUT_MS launchTimeoutMs: 90000, // BROWSER_LAUNCH_TIMEOUT_MS diff --git a/apps/docs/architecture/overview.md b/apps/docs/architecture/overview.md index fb1184c..734e28c 100644 --- a/apps/docs/architecture/overview.md +++ b/apps/docs/architecture/overview.md @@ -53,7 +53,7 @@ Maintains a fixed set of `{ browser, context }` pairs using [Camoufox](https://g Stores `{ cookies, userAgent, savedAt }` in Redis, keyed by hostname (`session:example.com`). The TTL is configurable (default 1 hour). Tier 3 writes to it on every successful challenge solve. Tier 2 reads from it at the start of every request. -Redis is optional — if `REDIS_URL` is not set, the session cache is disabled and every request escalates to Tier 3. +Redis is optional — if `REDIS_URL` is empty or not set, the session cache is disabled and every request escalates to Tier 3. ### Tiers (`packages/tiers`) diff --git a/apps/docs/architecture/session-cache.md b/apps/docs/architecture/session-cache.md index 21a017a..1ff4078 100644 --- a/apps/docs/architecture/session-cache.md +++ b/apps/docs/architecture/session-cache.md @@ -22,7 +22,7 @@ interface SessionData { } ``` -TTL: `SESSION_TTL_SECONDS` (default 3600 seconds / 1 hour). +TTL: `REDIS_SESSION_TTL_SECONDS` (default 3600 seconds / 1 hour). ## Session key @@ -42,7 +42,7 @@ Subdomains have separate sessions because WAF and application cookies can differ Tier 3 succeeds │ ├── extract cookies from browser context - ├── REDIS SET session:hostname → JSON EX SESSION_TTL_SECONDS + ├── REDIS SET session:hostname → JSON EX REDIS_SESSION_TTL_SECONDS │ └── next request to same domain: REDIS GET session:hostname @@ -63,10 +63,13 @@ This handles provider cookies expiring or being rejected before the Redis TTL en TRAWL's cache backend is Redis 8.8. TRAWL talks to it with `new RedisClient(REDIS_URL)` from Bun's native Redis client (not ioredis). +The cache is optional. When `REDIS_URL` is empty or unset, TRAWL does not create a Redis client and +Tier 2 remains disabled. + Each connection attempt is bounded by `REDIS_CONNECT_TIMEOUT_MS` (default 5 seconds). If Redis is not ready, scraping continues without Tier 2 while TRAWL retries in the background every -`REDIS_RETRY_DELAY_MS` (default 5 seconds). Set the retry delay to `0` when Redis is intentionally -absent. +`REDIS_RETRY_DELAY_MS` (default 5 seconds). Set the retry delay to `0` to disable reconnects after a +configured Redis endpoint becomes unavailable. ```typescript import { RedisClient } from 'bun' diff --git a/apps/docs/architecture/tiered-execution.md b/apps/docs/architecture/tiered-execution.md index 89e2f5a..7979228 100644 --- a/apps/docs/architecture/tiered-execution.md +++ b/apps/docs/architecture/tiered-execution.md @@ -57,7 +57,7 @@ Acquires a browser from the pool (or waits up to `BROWSER_ACQUIRE_TIMEOUT_MS` On success: - Extracts all cookies from the page context -- Writes `session:{hostname} → { cookies, userAgent, savedAt }` to Redis (TTL = `SESSION_TTL_SECONDS`) +- Writes `session:{hostname} → { cookies, userAgent, savedAt }` to Redis (TTL = `REDIS_SESSION_TTL_SECONDS`) - Returns the HTML and cookies to the caller Uses [Camoufox](https://github.com/daijro/camoufox) — Firefox with fingerprint patching at the C++/Juggler level to reduce common automation signals. Success still depends on the target's challenge variant, IP reputation, and upstream network conditions. diff --git a/apps/docs/deployment/configuration-migration.md b/apps/docs/deployment/configuration-migration.md new file mode 100644 index 0000000..f743291 --- /dev/null +++ b/apps/docs/deployment/configuration-migration.md @@ -0,0 +1,95 @@ +--- +title: Configuration Migration +description: Breaking environment-variable and Docker build-argument renames planned for the next TRAWL release. +--- + +# Configuration migration + +The next TRAWL release reorganizes configuration into short, subsystem-specific namespaces. The old +names are not retained as aliases: update `.env`, Compose overrides, container manifests, secrets, +and deployment automation before upgrading. + +## Environment variables + +| Previous name | New name | +| --- | --- | +| `SESSION_TTL_SECONDS` | `REDIS_SESSION_TTL_SECONDS` | +| `BROWSER_CONTENT_PROCESSES` | `BROWSER_MAX_CONTENT_PROCESSES` | +| `CAPTURE_MAX_CONSOLE_ENTRIES` | `DIAGNOSTICS_MAX_CONSOLE_ENTRIES` | +| `CAPTURE_MAX_NETWORK_ENTRIES` | `DIAGNOSTICS_MAX_NETWORK_ENTRIES` | +| `CAPTURE_SIZES_TIMEOUT_MS` | `DIAGNOSTICS_SIZE_TIMEOUT_MS` | +| `CAPTURE_MAX_REDIRECT_ENTRIES` | `REDIRECT_MAX_ENTRIES` | +| `CAPTURE_MAX_RESPONSE_PATTERNS` | `CAPTURE_MAX_PATTERNS` | +| `CAPTURE_MAX_RESPONSE_BYTES` | `CAPTURE_MAX_BODY_BYTES` | +| `CAPTURE_MAX_RESPONSE_TOTAL_BYTES` | `CAPTURE_MAX_TOTAL_BYTES` | +| `CAPTURE_SETTLE_IDLE_FLOOR_MS` | `CAPTURE_IDLE_FLOOR_MS` | +| `MITM_PROXY_ENABLED` | `MITM_ENABLED` | +| `MITM_PROXY_HOST` | `MITM_HOST` | +| `MITM_PROXY_PORT` | `MITM_PORT` | +| `MITM_PROXY_CA_DIR` | `MITM_CA_DIR` | +| `MITM_PROXY_MAX_TIER` | `MITM_MAX_TIER` | +| `MITM_PROXY_ALWAYS_SCRAPE` | `MITM_ALWAYS_SCRAPE` | +| `MITM_PROXY_DEBUG` | `MITM_DEBUG` | + +Two previous shared limits now have independent settings. Copy the old value to every applicable +new variable if you want to preserve custom limits exactly: + +| Previous shared name | New independent names | +| --- | --- | +| `CAPTURE_MAX_STRING_CHARS` | `DIAGNOSTICS_MAX_STRING_CHARS`, `REDIRECT_MAX_URL_CHARS`, `CAPTURE_MAX_METADATA_CHARS` | +| `CAPTURE_MAX_TOTAL_CHARS` | `DIAGNOSTICS_MAX_TOTAL_CHARS`, `REDIRECT_MAX_TOTAL_CHARS` | + +`CHROME_EXECUTABLE` was unused and has been removed. TRAWL uses its packaged Camoufox Firefox +binary; there is no replacement variable. + +## Redis opt-in behavior + +`REDIS_URL` is now the cache switch as well as the connection address: + +```text +empty or unset REDIS_URL → session cache disabled, no connection attempts +non-empty REDIS_URL → session cache enabled, background reconnects allowed +``` + +No separate enable flag is required. The standard Compose variants configure their bundled Redis; +the minimal variant defaults `REDIS_URL` to empty but can connect to an external Redis when given a URL. + +## Docker build argument + +The Camoufox-specific font-retention argument is now namespaced: + +```text +KEEP_SPOOFED_OS_FONTS → CAMOUFOX_KEEP_SPOOFED_OS_FONTS +``` + +For example: + +```bash +docker build \ + --build-arg CAMOUFOX_KEEP_SPOOFED_OS_FONTS=1 \ + -f apps/api/Dockerfile \ + -t trawl . +``` + +Keeping the spoofed macOS and Windows font bundles adds approximately 891 MB to the image. Enable +it only when rendered output must match those spoofed operating-system profiles. + +## Compose `.env` behavior + +The supplied Compose files explicitly pass every supported runtime setting from the adjacent `.env` +file into the TRAWL container. Unknown host variables are not forwarded. Run this after editing the +file to inspect the effective configuration: + +```bash +docker compose config +``` + +`PORT` is intentionally used only for the host-side `PORT:8191` mapping; the container continues to +listen on `8191`. The Redis-backed Compose variants route `REDIS_URL` to their bundled `redis` +service, while the minimal variant passes an optional external URL and defaults it to empty. + +Recreate the container after changing environment variables: + +```bash +docker compose up -d --force-recreate trawl +``` diff --git a/apps/docs/deployment/docker-compose.md b/apps/docs/deployment/docker-compose.md index c2c0654..cf5fd52 100644 --- a/apps/docs/deployment/docker-compose.md +++ b/apps/docs/deployment/docker-compose.md @@ -118,21 +118,23 @@ TRAWL and Redis. | `BROWSER_POOL_SIZE` | `3` | Warm browsers; supplied minimal/cached Compose files override this to `1` | | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000` | How long `acquire()` polls for a free browser before returning HTTP 429 | | `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8` | Rolling-replace after this many Tier 3/4 contexts; `0` disables it | +| `BROWSER_MAX_CONTENT_PROCESSES` | `2` | Maximum Firefox content processes per browser | | `REDIS_URL` | `redis://redis:6379` | Redis connection (set automatically in compose) | +| `REDIS_SESSION_TTL_SECONDS` | `3600` | Lifetime of cached sessions | | `REDIS_CONNECT_TIMEOUT_MS` | `5000` | Maximum time for each Redis connection attempt | | `REDIS_RETRY_DELAY_MS` | `5000` | Background reconnect delay; `0` disables retry | | `PROXY_URL` | — | Optional Tier 3 datacenter proxy or pool | | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation | -| `MITM_PROXY_ENABLED` | `false` | Starts the general HTTP/HTTPS proxy | -| `MITM_PROXY_PORT` | `8192` | Proxy listen and published port | -| `MITM_PROXY_HOST` | `0.0.0.0` | Proxy bind address | -| `MITM_PROXY_CA_DIR` | `/data/proxy-ca` | Persistent root CA directory | -| `MITM_PROXY_ALWAYS_SCRAPE` | `false` | Skip proxy Tier 0 and enter the scraper immediately | +| `MITM_ENABLED` | `false` | Starts the general HTTP/HTTPS proxy | +| `MITM_PORT` | `8192` | Proxy listen and published port | +| `MITM_HOST` | `0.0.0.0` | Proxy bind address | +| `MITM_CA_DIR` | `/data/proxy-ca` | Persistent root CA directory | +| `MITM_ALWAYS_SCRAPE` | `false` | Skip proxy Tier 0 and enter the scraper immediately | | `MCP_ENABLED` | `false` | Enables the Streamable HTTP endpoint at `/mcp` | | `MCP_ALLOWED_ORIGINS` | — | Comma-separated allowed browser origins | All supplied Compose files publish port `8192` and mount the `trawl_proxy_ca` volume. The listener -does not start until `MITM_PROXY_ENABLED=true`. See [Proxy Configuration](/proxy/configuration). +does not start until `MITM_ENABLED=true`. See [Proxy Configuration](/proxy/configuration). All supplied Compose files also pass `PROXY_URL`, `PROXY_LIST_FILE`, `RESIDENTIAL_PROXY_URL`, and `RESIDENTIAL_PROXY_LIST_FILE` from the local environment or `.env` file. For a single residential @@ -143,6 +145,17 @@ endpoint: RESIDENTIAL_PROXY_URL=http://user:pass@residential.example.com:8080 ``` +The files explicitly pass every supported TRAWL runtime variable, including browser, screenshot, +diagnostics, redirect, response-capture, STT, and ffmpeg tuning. Docker Compose uses `.env` for +interpolation but does not otherwise expose arbitrary host variables to the container. Inspect the +resolved values with `docker compose config` and see the +[configuration migration guide](/deployment/configuration-migration) when upgrading. + +There are two deliberate routing exceptions. `PORT` changes the published host port while the API +continues listening on container port `8191`. The Redis-backed variants always use their bundled +`redis` service; the minimal variant accepts an optional external `REDIS_URL` and otherwise disables +the cache. + For supported endpoint formats, pools, and mounted list files, see [Configuration → Proxies](/getting-started/configuration#proxies). diff --git a/apps/docs/deployment/standalone.md b/apps/docs/deployment/standalone.md index bbeb9ca..6faec6c 100644 --- a/apps/docs/deployment/standalone.md +++ b/apps/docs/deployment/standalone.md @@ -69,6 +69,20 @@ docker build -f apps/api/Dockerfile.baseline -t trawl:baseline . docker run -d --name trawl -p 8191:8191 --shm-size=1gb trawl:baseline ``` +Both API Dockerfiles omit the spoofed macOS and Windows font bundles by default. To retain them for +rendering that must match those Camoufox operating-system profiles, build with: + +```bash +docker build \ + --build-arg CAMOUFOX_KEEP_SPOOFED_OS_FONTS=1 \ + -f apps/api/Dockerfile \ + -t trawl . +``` + +This adds approximately 891 MB to the image. Other build arguments pin or validate bundled +dependencies and normally should not be overridden: `UBO_VERSION`, `UBO_AMO_FILE_ID`, `UBO_SHA256`, +`GEOLITE_CITY_MIN_BYTES`, and (for the baseline image) `BUN_VERSION`. + ::: warning Build context Both API Dockerfiles (`apps/api/Dockerfile` and `apps/api/Dockerfile.baseline`) require the **repo root** as the build context because they copy workspace packages (`packages/types`, `packages/browser`, `packages/tiers`). Always run `docker build` from the repo root with `-f apps/api/Dockerfile` (or `-f apps/api/Dockerfile.baseline`). ::: diff --git a/apps/docs/deployment/troubleshooting.md b/apps/docs/deployment/troubleshooting.md index ee2ce69..a8e765f 100644 --- a/apps/docs/deployment/troubleshooting.md +++ b/apps/docs/deployment/troubleshooting.md @@ -101,7 +101,7 @@ docker compose up -d --force-recreate **Causes:** 1. **Redis session data is not persisting** — Run `docker compose exec redis redis-cli keys "session:*"` after a successful scrape. If empty, the session cache write is failing. Check API logs for Redis connection errors. -2. **`SESSION_TTL_SECONDS` set too low** — If it's shorter than Cloudflare's challenge interval, the cache expires before the next request. +2. **`REDIS_SESSION_TTL_SECONDS` set too low** — If it's shorter than Cloudflare's challenge interval, the cache expires before the next request. 3. **Domain key mismatch** — The key is the hostname only. `sub.example.com` and `www.example.com` are separate sessions. ## POST /v1 returns HTTP 429 with `status: "error"` @@ -140,8 +140,8 @@ docker compose up -d --force-recreate First identify which TRAWL interface Prowlarr is using: - **FlareSolverr integration:** configure `http://trawl:8191` under **Settings → Indexers → - FlareSolverr**. Port `8191` is the API and does not require `MITM_PROXY_ENABLED`. -- **HTTP indexer proxy:** port `8192` is disabled by default. Set `MITM_PROXY_ENABLED=true`, recreate + FlareSolverr**. Port `8191` is the API and does not require `MITM_ENABLED`. +- **HTTP indexer proxy:** port `8192` is disabled by default. Set `MITM_ENABLED=true`, recreate the TRAWL service, and configure an HTTP proxy with host `trawl` and port `8192`. HTTPS targets also require TRAWL's CA in the Prowlarr container trust store; see [Proxy client setup](/proxy/client-setup#prowlarr). diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index 0cb3f58..7390db3 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -37,9 +37,10 @@ MCP_ALLOWED_ORIGINS=https://chat.example.com,https://admin.example.com ### `REDIS_URL` -**Default:** `redis://localhost:6379` +**Default:** _(empty — session cache disabled)_ -Standard Redis connection URL — TRAWL's cache backend is Redis 8.8. When running inside Docker Compose use the service name: +Standard Redis connection URL — TRAWL's cache backend is Redis 8.8. Set a non-empty URL to enable +the session cache. When running inside Docker Compose use the service name: ```ini REDIS_URL=redis://redis:6379 @@ -71,6 +72,22 @@ permanently disable Tier 2; TRAWL continues without the cache and reconnects in Delay between background connection attempts. Set it to `0` when Redis is intentionally absent to disable retries. The supplied minimal Compose variant does this automatically. +### `REDIS_SESSION_TTL_SECONDS` + +**Default:** `3600` (1 hour) + +How long solved browser cookies and user-agent state are cached in Redis per domain. After this TTL +the next protected request triggers a fresh challenge solve and refreshes the cache. + +Cloudflare's `cf_clearance` cookie typically has a 30-minute expiry. Setting +`REDIS_SESSION_TTL_SECONDS` below 1800 wastes cache hits; setting it above 7200 risks replaying +expired cookies. TRAWL handles an expired cookie by invalidating the cache and falling back to Tier 3. + +```ini +REDIS_SESSION_TTL_SECONDS=3600 # default — safe for most sites +REDIS_SESSION_TTL_SECONDS=1800 # more conservative +``` + ## Browser Pool ### `BROWSER_POOL_SIZE` @@ -114,15 +131,15 @@ BROWSER_RECYCLE_AFTER_CONTEXTS=8 # default - replace after 8 Tier 3/4 contexts BROWSER_RECYCLE_AFTER_CONTEXTS=0 # disable browser recycling entirely ``` -### `BROWSER_CONTENT_PROCESSES` +### `BROWSER_MAX_CONTENT_PROCESSES` **Default:** `2` Caps Firefox content processes per pooled browser via the `dom.ipc.processCount` Firefox pref. Firefox's default of 8 lets thread count climb when Tier 3 / Tier 4 churn disposable contexts (see #13). The cap bounds the leak at the source without paying the recycle cost. Raise if specific targets fail with empty content (rare). ```ini -BROWSER_CONTENT_PROCESSES=2 # default - conservative cap, lowest RAM/CPU -BROWSER_CONTENT_PROCESSES=4 # raise if CF/Imperva challenges stall +BROWSER_MAX_CONTENT_PROCESSES=2 # default - conservative cap, lowest RAM/CPU +BROWSER_MAX_CONTENT_PROCESSES=4 # raise if CF/Imperva challenges stall ``` ### `BROWSER_HEADFUL_POOL_SIZE` @@ -178,20 +195,19 @@ A screenshot is never worth failing a scrape: exceeding any of these bounds leav Tier 1 is a plain HTTP fetch and never captures a screenshot. Set `skipHttp: true` if you need to force a browser-tier attempt. -## Console, Network and Redirect Capture +## Console and Network Diagnostics -Only read when a request sets `consoleLogs`, `networkLogs` or `redirectChain` — see +Only read when a request sets `consoleLogs` or `networkLogs` — see [Native API](/api-reference/native-api). Without those flags no listener is attached and nothing is buffered. | Variable | Default | Purpose | | --- | ---: | --- | -| `CAPTURE_MAX_CONSOLE_ENTRIES` | `500` | Console messages kept per page | -| `CAPTURE_MAX_NETWORK_ENTRIES` | `1000` | Requests kept per page | -| `CAPTURE_MAX_STRING_CHARS` | `2000` | Longest single console message or URL kept | -| `CAPTURE_MAX_TOTAL_CHARS` | `1000000` | Total characters kept across both arrays | -| `CAPTURE_MAX_REDIRECT_ENTRIES` | `50` | URLs kept in the redirect chain | -| `CAPTURE_SIZES_TIMEOUT_MS` | `2000` | Maximum wait for the browser's per-request byte counts | +| `DIAGNOSTICS_MAX_CONSOLE_ENTRIES` | `500` | Console messages kept per page | +| `DIAGNOSTICS_MAX_NETWORK_ENTRIES` | `1000` | Requests kept per page | +| `DIAGNOSTICS_MAX_STRING_CHARS` | `2000` | Longest single console message or request URL kept | +| `DIAGNOSTICS_MAX_TOTAL_CHARS` | `1000000` | Total characters kept across both arrays | +| `DIAGNOSTICS_SIZE_TIMEOUT_MS` | `2000` | Maximum wait for the browser's per-request byte counts | Anything past a cap is dropped whole rather than truncated, and the number of dropped entries is logged once per scrape. A capture failure leaves the field unset and never @@ -201,6 +217,17 @@ Console messages and request URLs may contain credentials, tokens, personal data other sensitive values. Treat diagnostic fields as sensitive output and avoid storing or forwarding them unless necessary. +## Redirect Capture + +Only read when a request sets `redirectChain`. The tracker records only top-level document URLs; +subresource redirects are excluded. + +| Variable | Default | Purpose | +| --- | ---: | --- | +| `REDIRECT_MAX_ENTRIES` | `50` | URLs kept in the redirect chain | +| `REDIRECT_MAX_URL_CHARS` | `2000` | Longest individual redirect URL kept | +| `REDIRECT_MAX_TOTAL_CHARS` | `1000000` | Total characters kept across the redirect chain | + ## Response-Body Capture Only read when a request sets `captureResponses` — see @@ -209,15 +236,16 @@ body is read. | Variable | Default | Purpose | | --- | ---: | --- | -| `CAPTURE_MAX_RESPONSE_PATTERNS` | `10` | URL patterns honoured per request | +| `CAPTURE_MAX_PATTERNS` | `10` | URL patterns honoured per request | | `CAPTURE_MAX_RESPONSES` | `5` | Bodies kept per page, in arrival order | -| `CAPTURE_MAX_RESPONSE_BYTES` | `5242880` | Bytes kept per body; past it the body is trimmed and flagged `truncated` | -| `CAPTURE_MAX_RESPONSE_TOTAL_BYTES` | `10485760` | Bytes kept across all bodies of one page | +| `CAPTURE_MAX_BODY_BYTES` | `5242880` | Bytes kept per body; past it the body is trimmed and flagged `truncated` | +| `CAPTURE_MAX_TOTAL_BYTES` | `10485760` | Bytes kept across all bodies of one page | | `CAPTURE_MAX_READ_BYTES` | `10485760` | Largest body this process will read at all; a larger one is reported with an `error` instead of a trimmed prefix | +| `CAPTURE_MAX_METADATA_CHARS` | `2000` | Longest captured URL, header name, header value, or error kept | | `CAPTURE_BODY_TIMEOUT_MS` | `5000` | Maximum wait for in-flight body reads when the capture is drained | | `CAPTURE_SETTLE_MS` | `15000` | Default settle window when a request does not set `settleTimeout` | | `CAPTURE_MAX_SETTLE_MS` | `60000` | Ceiling a request may ask for; the request's own time budget also caps it | -| `CAPTURE_SETTLE_IDLE_FLOOR_MS` | `5000` | Network idle is ignored for this long, so a data fetch on a delayed timer is not mistaken for a quiet page | +| `CAPTURE_IDLE_FLOOR_MS` | `5000` | Network idle is ignored for this long, so a data fetch on a delayed timer is not mistaken for a quiet page | A response that matched but whose body could not be read is still returned, with `body` null and `error` set, so "nothing matched" stays distinguishable from "matched, retrieval @@ -229,21 +257,24 @@ bodies with a valid `Content-Length` are read. Compressed or unknown-size bodies returned with `body: null` and an error. Declared sizes are reserved cumulatively before reads start, so concurrent responses cannot exceed the total read budget. -## Session Cache +## CAPTCHA audio and media tools -### `SESSION_TTL_SECONDS` - -**Default:** `3600` (1 hour) +TRAWL uses ffmpeg while solving supported CAPTCHA challenges. reCAPTCHA audio is converted before +speech recognition, and the GeeTest solver uses it during image processing. -How long solved browser cookies and user-agent state are cached in Redis per domain. After this TTL -the next protected request triggers a fresh challenge solve (Tier 3) and refreshes the cache. +| Variable | Default | Purpose | +| --- | --- | --- | +| `STT_URL` | — | Optional Whisper/OpenAI-compatible transcription endpoint | +| `STT_API_KEY` | — | Optional bearer token sent only to `STT_URL` | +| `FFMPEG_PATH` | `ffmpeg` | Executable name or absolute path used by the CAPTCHA solvers | -Cloudflare's `cf_clearance` cookie typically has a 30-minute expiry. Setting `SESSION_TTL_SECONDS` below 1800 wastes cache hits; setting it above 7200 risks replaying expired cookies (TRAWL handles this gracefully by invalidating the cache and falling back to Tier 3). +Without `STT_URL`, reCAPTCHA audio uses Google's public speech-recognition endpoint. When `STT_URL` +is configured, TRAWL sends a multipart `whisper-1` transcription request and adds +`Authorization: Bearer ` when a key is present. -```ini -SESSION_TTL_SECONDS=3600 # default — safe for most sites -SESSION_TTL_SECONDS=1800 # more conservative -``` +The API container already includes ffmpeg. Bare-metal installations must make `ffmpeg` available on +`PATH` or set `FFMPEG_PATH`. These values are read only by CAPTCHA solving; ordinary scrapes do not +contact the configured STT service. Treat `STT_API_KEY` as a secret and avoid committing it to `.env`. ## Proxies @@ -374,10 +405,15 @@ PORT=9191 docker compose up -d ## Forward proxy The optional general HTTP/HTTPS proxy has its own listener, CA, tier cap, and debug settings. -See [Proxy Configuration](/proxy/configuration) for all `MITM_PROXY_*` variables and deployment +See [Proxy Configuration](/proxy/configuration) for all `MITM_*` variables and deployment examples. --- The repository's [`.env.example`](https://github.com/germondai/trawl/blob/main/.env.example) is the canonical copyable environment template. + +::: warning Upgrading from an earlier release? +The configuration namespaces changed without legacy aliases. Follow the complete +[configuration migration table](/deployment/configuration-migration) before recreating the container. +::: diff --git a/apps/docs/getting-started/quick-start.md b/apps/docs/getting-started/quick-start.md index f68accb..8298dcf 100644 --- a/apps/docs/getting-started/quick-start.md +++ b/apps/docs/getting-started/quick-start.md @@ -125,7 +125,7 @@ optional HTTP/HTTPS forward proxy and it is disabled by default. ``` localhost:8191 → TRAWL API (FlareSolverr-compatible + native endpoint) -localhost:8192 → HTTP/HTTPS forward proxy (when MITM_PROXY_ENABLED=true) +localhost:8192 → HTTP/HTTPS forward proxy (when MITM_ENABLED=true) ``` To stop everything: diff --git a/apps/docs/integrations/arr-apps.md b/apps/docs/integrations/arr-apps.md index 1ba6967..eb65b0a 100644 --- a/apps/docs/integrations/arr-apps.md +++ b/apps/docs/integrations/arr-apps.md @@ -56,5 +56,5 @@ Bazarr uses subtitle providers, not torrent indexers, so it does not use FlareSo | Repeat request with an accepted session | Tier 2 cached browser session | | Direct/datacenter path rejected | Tier 4, when a residential proxy is set | -The session cache TTL is configurable via `SESSION_TTL_SECONDS` (default 1 hour). Actual latency +The session cache TTL is configurable via `REDIS_SESSION_TTL_SECONDS` (default 1 hour). Actual latency depends on the indexer, challenge variant, IP reputation, and whether its saved session remains valid. diff --git a/apps/docs/integrations/prowlarr.md b/apps/docs/integrations/prowlarr.md index 130b14e..a9709b5 100644 --- a/apps/docs/integrations/prowlarr.md +++ b/apps/docs/integrations/prowlarr.md @@ -71,7 +71,7 @@ The API container isn't running or the port is wrong. Run `docker compose logs t If you configured TRAWL under **Settings → Indexer Proxies → HTTP** rather than as a FlareSolverr service, you are using the optional forward proxy. Enable it with -`MITM_PROXY_ENABLED=true`, recreate the `trawl` service, use port `8192`, and install TRAWL's CA in +`MITM_ENABLED=true`, recreate the `trawl` service, use port `8192`, and install TRAWL's CA in the Prowlarr container for HTTPS targets. See [Proxy client setup](/proxy/client-setup#prowlarr). **Test succeeds but searches still fail** diff --git a/apps/docs/proxy/ca-installation.md b/apps/docs/proxy/ca-installation.md index 427913b..1a84753 100644 --- a/apps/docs/proxy/ca-installation.md +++ b/apps/docs/proxy/ca-installation.md @@ -11,7 +11,7 @@ creates: - `ca.crt` — the root certificate clients install; - `ca.key` — the private signing key, which must remain secret. -Both files live in `MITM_PROXY_CA_DIR`. Per-host certificates are generated in memory and signed by +Both files live in `MITM_CA_DIR`. Per-host certificates are generated in memory and signed by this root. Persist the directory so clients only need to install the root once. ::: danger diff --git a/apps/docs/proxy/client-setup.md b/apps/docs/proxy/client-setup.md index 50b1292..198ec76 100644 --- a/apps/docs/proxy/client-setup.md +++ b/apps/docs/proxy/client-setup.md @@ -102,5 +102,5 @@ Avoid setting these globally inside the TRAWL container itself; that can create | WebSocket handshake is rejected | The direct handshake lacked required cookies or authorization | | Proxy works in curl but not the application | The application uses a separate trust store or ignores system proxy settings | -Enable `MITM_PROXY_DEBUG=true`, reproduce one request, and inspect the TRAWL logs to see whether it +Enable `MITM_DEBUG=true`, reproduce one request, and inspect the TRAWL logs to see whether it used Tier 0, streaming, or the scrape fallback. diff --git a/apps/docs/proxy/configuration.md b/apps/docs/proxy/configuration.md index 3b3b73d..42f7514 100644 --- a/apps/docs/proxy/configuration.md +++ b/apps/docs/proxy/configuration.md @@ -9,33 +9,33 @@ description: Enable and configure TRAWL's challenge-bypassing HTTP/HTTPS proxy. | Variable | Default | Purpose | | -------------------------- | ---------------- | -------------------------------------------------------- | -| `MITM_PROXY_ENABLED` | `false` | Starts the forward-proxy listener | -| `MITM_PROXY_HOST` | `0.0.0.0` | Listener bind address | -| `MITM_PROXY_PORT` | `8192` | Listener port | -| `MITM_PROXY_CA_DIR` | `/data/proxy-ca` | Persistent root CA certificate and private-key directory | -| `MITM_PROXY_MAX_TIER` | `4` | Highest solver tier available to proxy escalation | -| `MITM_PROXY_ALWAYS_SCRAPE` | `false` | Skip direct Tier 0 and enter the scraper immediately | -| `MITM_PROXY_DEBUG` | `false` | Logs proxied requests and tier attempts | +| `MITM_ENABLED` | `false` | Starts the forward-proxy listener | +| `MITM_HOST` | `0.0.0.0` | Listener bind address | +| `MITM_PORT` | `8192` | Listener port | +| `MITM_CA_DIR` | `/data/proxy-ca` | Persistent root CA certificate and private-key directory | +| `MITM_MAX_TIER` | `4` | Highest solver tier available to proxy escalation | +| `MITM_ALWAYS_SCRAPE` | `false` | Skip direct Tier 0 and enter the scraper immediately | +| `MITM_DEBUG` | `false` | Logs proxied requests and tier attempts | Example: ```ini -MITM_PROXY_ENABLED=true -MITM_PROXY_HOST=127.0.0.1 -MITM_PROXY_PORT=8192 -MITM_PROXY_CA_DIR=/data/proxy-ca -MITM_PROXY_MAX_TIER=4 -MITM_PROXY_ALWAYS_SCRAPE=false -MITM_PROXY_DEBUG=false +MITM_ENABLED=true +MITM_HOST=127.0.0.1 +MITM_PORT=8192 +MITM_CA_DIR=/data/proxy-ca +MITM_MAX_TIER=4 +MITM_ALWAYS_SCRAPE=false +MITM_DEBUG=false ``` Use `127.0.0.1` for a local installation. Docker clients on a bridge network normally require `0.0.0.0`; restrict access with container networking or a host firewall. -`MITM_PROXY_MAX_TIER=3` prevents proxy requests from consuming a configured residential Tier 4 +`MITM_MAX_TIER=3` prevents proxy requests from consuming a configured residential Tier 4 proxy. An empty or invalid value uses the normal maximum of Tier 4. -Set `MITM_PROXY_ALWAYS_SCRAPE=true` for targets where the proxy's initial direct Tier 0 request is +Set `MITM_ALWAYS_SCRAPE=true` for targets where the proxy's initial direct Tier 0 request is itself enough to trigger a temporary ban. This skips only proxy Tier 0: the normal scraper ladder still starts at its Tier 1 plain fetch and escalates when necessary. WebSocket upgrades remain direct relays. @@ -57,11 +57,11 @@ services: - "8191:8191" - "8192:8192" environment: - MITM_PROXY_ENABLED: "true" - MITM_PROXY_HOST: 0.0.0.0 - MITM_PROXY_PORT: 8192 - MITM_PROXY_CA_DIR: /data/proxy-ca - MITM_PROXY_ALWAYS_SCRAPE: "false" + MITM_ENABLED: "true" + MITM_HOST: 0.0.0.0 + MITM_PORT: 8192 + MITM_CA_DIR: /data/proxy-ca + MITM_ALWAYS_SCRAPE: "false" volumes: - trawl_proxy_ca:/data/proxy-ca @@ -82,7 +82,7 @@ Tier 0 direct traffic leaves from the TRAWL host directly. `PROXY_URL` and not turn the entire forward proxy into a chain through another proxy. The normal sticky-per-domain rotation and failure cooldown rules apply during escalation. Use -`MITM_PROXY_MAX_TIER` to cap which tiers the forward proxy may reach. +`MITM_MAX_TIER` to cap which tiers the forward proxy may reach. ## Verify the listener @@ -114,7 +114,7 @@ The Range request should return `206` and a 100-byte body when the upstream supp ## Debug logging -Set `MITM_PROXY_DEBUG=true` to log direct forwarding, streaming decisions, challenge escalation, +Set `MITM_DEBUG=true` to log direct forwarding, streaming decisions, challenge escalation, winning scrape tiers, statuses, content types, and payload sizes. Disable it after troubleshooting; general proxy clients can generate a large volume of requests. diff --git a/apps/web/app/components/CtaSection.vue b/apps/web/app/components/CtaSection.vue index 958ef06..80ad08a 100644 --- a/apps/web/app/components/CtaSection.vue +++ b/apps/web/app/components/CtaSection.vue @@ -70,12 +70,12 @@ const tabs: { id: Tab; label: string; hint: string }[] = [ image: ghcr.io/germondai/trawl:latest ports: - "${PORT:-8191}:8191" - - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}" + - "${MITM_PORT:-8192}:${MITM_PORT:-8192}" shm_size: 1gb environment: BROWSER_POOL_SIZE: 1 - MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} - MITM_PROXY_CA_DIR: /data/proxy-ca + MITM_ENABLED: ${MITM_ENABLED:-false} + MITM_CA_DIR: /data/proxy-ca volumes: - trawl_proxy_ca:/data/proxy-ca healthcheck: @@ -98,13 +98,13 @@ const tabs: { id: Tab; label: string; hint: string }[] = [ image: ghcr.io/germondai/trawl:latest ports: - "${PORT:-8191}:8191" - - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}" + - "${MITM_PORT:-8192}:${MITM_PORT:-8192}" shm_size: 1gb environment: REDIS_URL: redis://redis:6379 BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-1} - MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} - MITM_PROXY_CA_DIR: /data/proxy-ca + MITM_ENABLED: ${MITM_ENABLED:-false} + MITM_CA_DIR: /data/proxy-ca volumes: - trawl_proxy_ca:/data/proxy-ca depends_on: @@ -131,14 +131,14 @@ const tabs: { id: Tab; label: string; hint: string }[] = [ restart: always ports: - "${PORT:-8191}:8191" - - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}" + - "${MITM_PORT:-8192}:${MITM_PORT:-8192}" shm_size: 1gb mem_limit: 3g environment: REDIS_URL: redis://redis:6379 BROWSER_POOL_SIZE: 3 - MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} - MITM_PROXY_CA_DIR: /data/proxy-ca + MITM_ENABLED: ${MITM_ENABLED:-false} + MITM_CA_DIR: /data/proxy-ca volumes: - trawl_proxy_ca:/data/proxy-ca depends_on: diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index ce9f6ca..f309531 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -3,29 +3,63 @@ services: image: ghcr.io/germondai/trawl:latest ports: - "${PORT:-8191}:8191" - # MITM proxy (only when MITM_PROXY_ENABLED=true). CA cert at :8191/proxy-ca.crt - - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}" + # MITM proxy (only when MITM_ENABLED=true). CA cert at :8191/proxy-ca.crt + - "${MITM_PORT:-8192}:${MITM_PORT:-8192}" shm_size: 1gb environment: - # This variant has no Redis service, so do not keep retrying after the initial cache probe. + # This variant has no bundled Redis; an empty URL keeps the cache disabled. + REDIS_URL: ${REDIS_URL:-} + REDIS_SESSION_TTL_SECONDS: ${REDIS_SESSION_TTL_SECONDS:-3600} + REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-5000} REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-0} BROWSER_POOL_SIZE: 1 + BROWSER_MAX_CONTENT_PROCESSES: ${BROWSER_MAX_CONTENT_PROCESSES:-2} BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0} + BROWSER_ACQUIRE_TIMEOUT_MS: ${BROWSER_ACQUIRE_TIMEOUT_MS:-15000} + BROWSER_RECYCLE_AFTER_CONTEXTS: ${BROWSER_RECYCLE_AFTER_CONTEXTS:-8} + BROWSER_STALL_TIMEOUT_MS: ${BROWSER_STALL_TIMEOUT_MS:-180000} + BROWSER_CLOSE_TIMEOUT_MS: ${BROWSER_CLOSE_TIMEOUT_MS:-10000} + BROWSER_LAUNCH_TIMEOUT_MS: ${BROWSER_LAUNCH_TIMEOUT_MS:-90000} + SCREENSHOT_SETTLE_MS: ${SCREENSHOT_SETTLE_MS:-3000} + SCREENSHOT_TIMEOUT_MS: ${SCREENSHOT_TIMEOUT_MS:-10000} + SCREENSHOT_JPEG_QUALITY: ${SCREENSHOT_JPEG_QUALITY:-60} + SCREENSHOT_MAX_BYTES: ${SCREENSHOT_MAX_BYTES:-4000000} + DIAGNOSTICS_MAX_CONSOLE_ENTRIES: ${DIAGNOSTICS_MAX_CONSOLE_ENTRIES:-500} + DIAGNOSTICS_MAX_NETWORK_ENTRIES: ${DIAGNOSTICS_MAX_NETWORK_ENTRIES:-1000} + DIAGNOSTICS_MAX_STRING_CHARS: ${DIAGNOSTICS_MAX_STRING_CHARS:-2000} + DIAGNOSTICS_MAX_TOTAL_CHARS: ${DIAGNOSTICS_MAX_TOTAL_CHARS:-1000000} + DIAGNOSTICS_SIZE_TIMEOUT_MS: ${DIAGNOSTICS_SIZE_TIMEOUT_MS:-2000} + REDIRECT_MAX_ENTRIES: ${REDIRECT_MAX_ENTRIES:-50} + REDIRECT_MAX_URL_CHARS: ${REDIRECT_MAX_URL_CHARS:-2000} + REDIRECT_MAX_TOTAL_CHARS: ${REDIRECT_MAX_TOTAL_CHARS:-1000000} + CAPTURE_MAX_PATTERNS: ${CAPTURE_MAX_PATTERNS:-10} + CAPTURE_MAX_RESPONSES: ${CAPTURE_MAX_RESPONSES:-5} + CAPTURE_MAX_BODY_BYTES: ${CAPTURE_MAX_BODY_BYTES:-5242880} + CAPTURE_MAX_TOTAL_BYTES: ${CAPTURE_MAX_TOTAL_BYTES:-10485760} + CAPTURE_MAX_READ_BYTES: ${CAPTURE_MAX_READ_BYTES:-10485760} + CAPTURE_MAX_METADATA_CHARS: ${CAPTURE_MAX_METADATA_CHARS:-2000} + CAPTURE_BODY_TIMEOUT_MS: ${CAPTURE_BODY_TIMEOUT_MS:-5000} + CAPTURE_SETTLE_MS: ${CAPTURE_SETTLE_MS:-15000} + CAPTURE_MAX_SETTLE_MS: ${CAPTURE_MAX_SETTLE_MS:-60000} + CAPTURE_IDLE_FLOOR_MS: ${CAPTURE_IDLE_FLOOR_MS:-5000} + STT_URL: ${STT_URL:-} + STT_API_KEY: ${STT_API_KEY:-} + FFMPEG_PATH: ${FFMPEG_PATH:-} MCP_ENABLED: ${MCP_ENABLED:-false} MCP_ALLOWED_ORIGINS: ${MCP_ALLOWED_ORIGINS:-} PROXY_URL: ${PROXY_URL:-} PROXY_LIST_FILE: ${PROXY_LIST_FILE:-} RESIDENTIAL_PROXY_URL: ${RESIDENTIAL_PROXY_URL:-} RESIDENTIAL_PROXY_LIST_FILE: ${RESIDENTIAL_PROXY_LIST_FILE:-} - MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} - MITM_PROXY_HOST: ${MITM_PROXY_HOST:-0.0.0.0} - MITM_PROXY_PORT: ${MITM_PROXY_PORT:-8192} - MITM_PROXY_CA_DIR: ${MITM_PROXY_CA_DIR:-/data/proxy-ca} - MITM_PROXY_MAX_TIER: ${MITM_PROXY_MAX_TIER:-4} - MITM_PROXY_ALWAYS_SCRAPE: ${MITM_PROXY_ALWAYS_SCRAPE:-false} - MITM_PROXY_DEBUG: ${MITM_PROXY_DEBUG:-false} + MITM_ENABLED: ${MITM_ENABLED:-false} + MITM_HOST: ${MITM_HOST:-0.0.0.0} + MITM_PORT: ${MITM_PORT:-8192} + MITM_CA_DIR: ${MITM_CA_DIR:-/data/proxy-ca} + MITM_MAX_TIER: ${MITM_MAX_TIER:-4} + MITM_ALWAYS_SCRAPE: ${MITM_ALWAYS_SCRAPE:-false} + MITM_DEBUG: ${MITM_DEBUG:-false} volumes: - - trawl_proxy_ca:${MITM_PROXY_CA_DIR:-/data/proxy-ca} + - trawl_proxy_ca:${MITM_CA_DIR:-/data/proxy-ca} healthcheck: test: ["CMD", "curl", "-sf", "http://localhost:8191/health"] interval: 30s diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 7bf67f6..42bd6dc 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -16,31 +16,63 @@ services: restart: always ports: - "${PORT:-8191}:8191" - # MITM proxy (only when MITM_PROXY_ENABLED=true). CA cert at :8191/proxy-ca.crt - - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}" + # MITM proxy (only when MITM_ENABLED=true). CA cert at :8191/proxy-ca.crt + - "${MITM_PORT:-8192}:${MITM_PORT:-8192}" shm_size: 1gb mem_limit: 3g environment: REDIS_URL: redis://redis:6379 + REDIS_SESSION_TTL_SECONDS: ${REDIS_SESSION_TTL_SECONDS:-3600} REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-5000} REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-5000} BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-3} + BROWSER_MAX_CONTENT_PROCESSES: ${BROWSER_MAX_CONTENT_PROCESSES:-2} BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0} + BROWSER_ACQUIRE_TIMEOUT_MS: ${BROWSER_ACQUIRE_TIMEOUT_MS:-15000} + BROWSER_RECYCLE_AFTER_CONTEXTS: ${BROWSER_RECYCLE_AFTER_CONTEXTS:-8} + BROWSER_STALL_TIMEOUT_MS: ${BROWSER_STALL_TIMEOUT_MS:-180000} + BROWSER_CLOSE_TIMEOUT_MS: ${BROWSER_CLOSE_TIMEOUT_MS:-10000} + BROWSER_LAUNCH_TIMEOUT_MS: ${BROWSER_LAUNCH_TIMEOUT_MS:-90000} + SCREENSHOT_SETTLE_MS: ${SCREENSHOT_SETTLE_MS:-3000} + SCREENSHOT_TIMEOUT_MS: ${SCREENSHOT_TIMEOUT_MS:-10000} + SCREENSHOT_JPEG_QUALITY: ${SCREENSHOT_JPEG_QUALITY:-60} + SCREENSHOT_MAX_BYTES: ${SCREENSHOT_MAX_BYTES:-4000000} + DIAGNOSTICS_MAX_CONSOLE_ENTRIES: ${DIAGNOSTICS_MAX_CONSOLE_ENTRIES:-500} + DIAGNOSTICS_MAX_NETWORK_ENTRIES: ${DIAGNOSTICS_MAX_NETWORK_ENTRIES:-1000} + DIAGNOSTICS_MAX_STRING_CHARS: ${DIAGNOSTICS_MAX_STRING_CHARS:-2000} + DIAGNOSTICS_MAX_TOTAL_CHARS: ${DIAGNOSTICS_MAX_TOTAL_CHARS:-1000000} + DIAGNOSTICS_SIZE_TIMEOUT_MS: ${DIAGNOSTICS_SIZE_TIMEOUT_MS:-2000} + REDIRECT_MAX_ENTRIES: ${REDIRECT_MAX_ENTRIES:-50} + REDIRECT_MAX_URL_CHARS: ${REDIRECT_MAX_URL_CHARS:-2000} + REDIRECT_MAX_TOTAL_CHARS: ${REDIRECT_MAX_TOTAL_CHARS:-1000000} + CAPTURE_MAX_PATTERNS: ${CAPTURE_MAX_PATTERNS:-10} + CAPTURE_MAX_RESPONSES: ${CAPTURE_MAX_RESPONSES:-5} + CAPTURE_MAX_BODY_BYTES: ${CAPTURE_MAX_BODY_BYTES:-5242880} + CAPTURE_MAX_TOTAL_BYTES: ${CAPTURE_MAX_TOTAL_BYTES:-10485760} + CAPTURE_MAX_READ_BYTES: ${CAPTURE_MAX_READ_BYTES:-10485760} + CAPTURE_MAX_METADATA_CHARS: ${CAPTURE_MAX_METADATA_CHARS:-2000} + CAPTURE_BODY_TIMEOUT_MS: ${CAPTURE_BODY_TIMEOUT_MS:-5000} + CAPTURE_SETTLE_MS: ${CAPTURE_SETTLE_MS:-15000} + CAPTURE_MAX_SETTLE_MS: ${CAPTURE_MAX_SETTLE_MS:-60000} + CAPTURE_IDLE_FLOOR_MS: ${CAPTURE_IDLE_FLOOR_MS:-5000} + STT_URL: ${STT_URL:-} + STT_API_KEY: ${STT_API_KEY:-} + FFMPEG_PATH: ${FFMPEG_PATH:-} MCP_ENABLED: ${MCP_ENABLED:-false} MCP_ALLOWED_ORIGINS: ${MCP_ALLOWED_ORIGINS:-} PROXY_URL: ${PROXY_URL:-} PROXY_LIST_FILE: ${PROXY_LIST_FILE:-} RESIDENTIAL_PROXY_URL: ${RESIDENTIAL_PROXY_URL:-} RESIDENTIAL_PROXY_LIST_FILE: ${RESIDENTIAL_PROXY_LIST_FILE:-} - MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} - MITM_PROXY_HOST: ${MITM_PROXY_HOST:-0.0.0.0} - MITM_PROXY_PORT: ${MITM_PROXY_PORT:-8192} - MITM_PROXY_CA_DIR: ${MITM_PROXY_CA_DIR:-/data/proxy-ca} - MITM_PROXY_MAX_TIER: ${MITM_PROXY_MAX_TIER:-4} - MITM_PROXY_ALWAYS_SCRAPE: ${MITM_PROXY_ALWAYS_SCRAPE:-false} - MITM_PROXY_DEBUG: ${MITM_PROXY_DEBUG:-false} + MITM_ENABLED: ${MITM_ENABLED:-false} + MITM_HOST: ${MITM_HOST:-0.0.0.0} + MITM_PORT: ${MITM_PORT:-8192} + MITM_CA_DIR: ${MITM_CA_DIR:-/data/proxy-ca} + MITM_MAX_TIER: ${MITM_MAX_TIER:-4} + MITM_ALWAYS_SCRAPE: ${MITM_ALWAYS_SCRAPE:-false} + MITM_DEBUG: ${MITM_DEBUG:-false} volumes: - - trawl_proxy_ca:${MITM_PROXY_CA_DIR:-/data/proxy-ca} + - trawl_proxy_ca:${MITM_CA_DIR:-/data/proxy-ca} depends_on: redis: condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 7e59689..e29c5a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,17 +15,49 @@ services: ports: # API + FlareSolverr-compatible /v1 - "${PORT:-8191}:8191" - # MITM forward proxy (:8192) — only used when MITM_PROXY_ENABLED=true. + # MITM forward proxy (:8192) — only used when MITM_ENABLED=true. # The CA cert is served from :8191/proxy-ca.crt and must be installed # in each client's trust store (system keychain, Java cacerts, etc.). - - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}" + - "${MITM_PORT:-8192}:${MITM_PORT:-8192}" shm_size: 1gb environment: REDIS_URL: redis://redis:6379 + REDIS_SESSION_TTL_SECONDS: ${REDIS_SESSION_TTL_SECONDS:-3600} REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-5000} REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-5000} BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-1} + BROWSER_MAX_CONTENT_PROCESSES: ${BROWSER_MAX_CONTENT_PROCESSES:-2} BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0} + BROWSER_ACQUIRE_TIMEOUT_MS: ${BROWSER_ACQUIRE_TIMEOUT_MS:-15000} + BROWSER_RECYCLE_AFTER_CONTEXTS: ${BROWSER_RECYCLE_AFTER_CONTEXTS:-8} + BROWSER_STALL_TIMEOUT_MS: ${BROWSER_STALL_TIMEOUT_MS:-180000} + BROWSER_CLOSE_TIMEOUT_MS: ${BROWSER_CLOSE_TIMEOUT_MS:-10000} + BROWSER_LAUNCH_TIMEOUT_MS: ${BROWSER_LAUNCH_TIMEOUT_MS:-90000} + SCREENSHOT_SETTLE_MS: ${SCREENSHOT_SETTLE_MS:-3000} + SCREENSHOT_TIMEOUT_MS: ${SCREENSHOT_TIMEOUT_MS:-10000} + SCREENSHOT_JPEG_QUALITY: ${SCREENSHOT_JPEG_QUALITY:-60} + SCREENSHOT_MAX_BYTES: ${SCREENSHOT_MAX_BYTES:-4000000} + DIAGNOSTICS_MAX_CONSOLE_ENTRIES: ${DIAGNOSTICS_MAX_CONSOLE_ENTRIES:-500} + DIAGNOSTICS_MAX_NETWORK_ENTRIES: ${DIAGNOSTICS_MAX_NETWORK_ENTRIES:-1000} + DIAGNOSTICS_MAX_STRING_CHARS: ${DIAGNOSTICS_MAX_STRING_CHARS:-2000} + DIAGNOSTICS_MAX_TOTAL_CHARS: ${DIAGNOSTICS_MAX_TOTAL_CHARS:-1000000} + DIAGNOSTICS_SIZE_TIMEOUT_MS: ${DIAGNOSTICS_SIZE_TIMEOUT_MS:-2000} + REDIRECT_MAX_ENTRIES: ${REDIRECT_MAX_ENTRIES:-50} + REDIRECT_MAX_URL_CHARS: ${REDIRECT_MAX_URL_CHARS:-2000} + REDIRECT_MAX_TOTAL_CHARS: ${REDIRECT_MAX_TOTAL_CHARS:-1000000} + CAPTURE_MAX_PATTERNS: ${CAPTURE_MAX_PATTERNS:-10} + CAPTURE_MAX_RESPONSES: ${CAPTURE_MAX_RESPONSES:-5} + CAPTURE_MAX_BODY_BYTES: ${CAPTURE_MAX_BODY_BYTES:-5242880} + CAPTURE_MAX_TOTAL_BYTES: ${CAPTURE_MAX_TOTAL_BYTES:-10485760} + CAPTURE_MAX_READ_BYTES: ${CAPTURE_MAX_READ_BYTES:-10485760} + CAPTURE_MAX_METADATA_CHARS: ${CAPTURE_MAX_METADATA_CHARS:-2000} + CAPTURE_BODY_TIMEOUT_MS: ${CAPTURE_BODY_TIMEOUT_MS:-5000} + CAPTURE_SETTLE_MS: ${CAPTURE_SETTLE_MS:-15000} + CAPTURE_MAX_SETTLE_MS: ${CAPTURE_MAX_SETTLE_MS:-60000} + CAPTURE_IDLE_FLOOR_MS: ${CAPTURE_IDLE_FLOOR_MS:-5000} + STT_URL: ${STT_URL:-} + STT_API_KEY: ${STT_API_KEY:-} + FFMPEG_PATH: ${FFMPEG_PATH:-} MCP_ENABLED: ${MCP_ENABLED:-false} MCP_ALLOWED_ORIGINS: ${MCP_ALLOWED_ORIGINS:-} PROXY_URL: ${PROXY_URL:-} @@ -33,16 +65,16 @@ services: RESIDENTIAL_PROXY_URL: ${RESIDENTIAL_PROXY_URL:-} RESIDENTIAL_PROXY_LIST_FILE: ${RESIDENTIAL_PROXY_LIST_FILE:-} # MITM proxy (off by default — opt in via .env) - MITM_PROXY_ENABLED: ${MITM_PROXY_ENABLED:-false} - MITM_PROXY_HOST: ${MITM_PROXY_HOST:-0.0.0.0} - MITM_PROXY_PORT: ${MITM_PROXY_PORT:-8192} - MITM_PROXY_CA_DIR: ${MITM_PROXY_CA_DIR:-/data/proxy-ca} - MITM_PROXY_MAX_TIER: ${MITM_PROXY_MAX_TIER:-4} - MITM_PROXY_ALWAYS_SCRAPE: ${MITM_PROXY_ALWAYS_SCRAPE:-false} - MITM_PROXY_DEBUG: ${MITM_PROXY_DEBUG:-false} + MITM_ENABLED: ${MITM_ENABLED:-false} + MITM_HOST: ${MITM_HOST:-0.0.0.0} + MITM_PORT: ${MITM_PORT:-8192} + MITM_CA_DIR: ${MITM_CA_DIR:-/data/proxy-ca} + MITM_MAX_TIER: ${MITM_MAX_TIER:-4} + MITM_ALWAYS_SCRAPE: ${MITM_ALWAYS_SCRAPE:-false} + MITM_DEBUG: ${MITM_DEBUG:-false} volumes: # Persist the proxy's CA so clients don't need to re-install on every restart. - - trawl_proxy_ca:${MITM_PROXY_CA_DIR:-/data/proxy-ca} + - trawl_proxy_ca:${MITM_CA_DIR:-/data/proxy-ca} depends_on: redis: condition: service_healthy diff --git a/packages/tiers/src/screenshot.ts b/packages/tiers/src/screenshot.ts index 116e75a..3f3a058 100644 --- a/packages/tiers/src/screenshot.ts +++ b/packages/tiers/src/screenshot.ts @@ -1,12 +1,14 @@ import type { Page } from "patchright" +import { captureLimit } from "./utils/captureConfig" // A screenshot is a best-effort side artifact of a scrape, so every step is bounded: // the settle wait, the capture itself, and the size of the image we are willing to // carry in the response. All are env-tunable. -const SETTLE_MS = Number(process.env.SCREENSHOT_SETTLE_MS ?? 3_000) -const CAPTURE_TIMEOUT_MS = Number(process.env.SCREENSHOT_TIMEOUT_MS ?? 10_000) -const JPEG_QUALITY = Number(process.env.SCREENSHOT_JPEG_QUALITY ?? 60) -const MAX_BYTES = Number(process.env.SCREENSHOT_MAX_BYTES ?? 4_000_000) +const SETTLE_MS = captureLimit(process.env.SCREENSHOT_SETTLE_MS, 3_000) +const CAPTURE_TIMEOUT_MS = captureLimit(process.env.SCREENSHOT_TIMEOUT_MS, 10_000) +const configuredQuality = captureLimit(process.env.SCREENSHOT_JPEG_QUALITY, 60) +const JPEG_QUALITY = configuredQuality >= 1 && configuredQuality <= 100 ? configuredQuality : 60 +const MAX_BYTES = captureLimit(process.env.SCREENSHOT_MAX_BYTES, 4_000_000) // Viewport only — never fullPage. A challenge wall or an infinite-scroll page stitches // into a tall, mostly-empty canvas that costs seconds and shows less than the first diff --git a/packages/tiers/src/solvers/geetest.ts b/packages/tiers/src/solvers/geetest.ts index 04d5aec..e3f330b 100644 --- a/packages/tiers/src/solvers/geetest.ts +++ b/packages/tiers/src/solvers/geetest.ts @@ -11,7 +11,7 @@ import { randomUUID } from "node:crypto" import { $ } from "bun" import type { Page } from "patchright" -const FFMPEG = process.env.FFMPEG_PATH ?? "ffmpeg" +const FFMPEG = process.env.FFMPEG_PATH?.trim() || "ffmpeg" // Initial "Click to verify" button selectors (GeeTest v4 entry point). // Use aria-label and specific class — avoid [class*="geetest_btn"] which also matches the icon SVG. diff --git a/packages/tiers/src/solvers/stt.ts b/packages/tiers/src/solvers/stt.ts index 5e0a33e..e4465ce 100644 --- a/packages/tiers/src/solvers/stt.ts +++ b/packages/tiers/src/solvers/stt.ts @@ -14,12 +14,12 @@ import { randomUUID } from "node:crypto" import { $ } from "bun" -const STT_URL = process.env.STT_URL ?? "" +const STT_URL = process.env.STT_URL?.trim() ?? "" const STT_KEY = process.env.STT_API_KEY ?? "" // FFMPEG_PATH: full path to ffmpeg binary. Docker installs 'ffmpeg' via apt. // On macOS with Playwright's bundled binary it's named 'ffmpeg-mac'; set this // env var or create a symlink to make 'ffmpeg' resolve. -const FFMPEG = process.env.FFMPEG_PATH ?? "ffmpeg" +const FFMPEG = process.env.FFMPEG_PATH?.trim() || "ffmpeg" // Google's public Speech API key — used in Google's own demos and the Buster extension. // Has been public since 2013. Google can't revoke it without breaking their own accessibility tooling. diff --git a/packages/tiers/src/utils/capture.ts b/packages/tiers/src/utils/capture.ts index b7cb663..c5b600c 100644 --- a/packages/tiers/src/utils/capture.ts +++ b/packages/tiers/src/utils/capture.ts @@ -8,11 +8,11 @@ import { attachResponseCapture, type ResponseCaptureOptions } from "./responseCa // bounded: entry counts, the length of any single captured string, and the total across // both arrays. Anything past a cap is dropped whole rather than silently truncated, and // the drop count is logged when the capture is drained. All caps are env-tunable. -const MAX_CONSOLE_ENTRIES = captureLimit(process.env.CAPTURE_MAX_CONSOLE_ENTRIES, 500) -const MAX_NETWORK_ENTRIES = captureLimit(process.env.CAPTURE_MAX_NETWORK_ENTRIES, 1_000) -const MAX_STRING_CHARS = captureLimit(process.env.CAPTURE_MAX_STRING_CHARS, 2_000) -const MAX_TOTAL_CHARS = captureLimit(process.env.CAPTURE_MAX_TOTAL_CHARS, 1_000_000) -const SIZES_TIMEOUT_MS = captureLimit(process.env.CAPTURE_SIZES_TIMEOUT_MS, 2_000) +const MAX_CONSOLE_ENTRIES = captureLimit(process.env.DIAGNOSTICS_MAX_CONSOLE_ENTRIES, 500) +const MAX_NETWORK_ENTRIES = captureLimit(process.env.DIAGNOSTICS_MAX_NETWORK_ENTRIES, 1_000) +const MAX_STRING_CHARS = captureLimit(process.env.DIAGNOSTICS_MAX_STRING_CHARS, 2_000) +const MAX_TOTAL_CHARS = captureLimit(process.env.DIAGNOSTICS_MAX_TOTAL_CHARS, 1_000_000) +const SIZES_TIMEOUT_MS = captureLimit(process.env.DIAGNOSTICS_SIZE_TIMEOUT_MS, 2_000) // Console types that carry a severity of their own; everything else is informational. const CONSOLE_LEVELS: Record = { diff --git a/packages/tiers/src/utils/mainResponse.ts b/packages/tiers/src/utils/mainResponse.ts index 48ab760..4d97762 100644 --- a/packages/tiers/src/utils/mainResponse.ts +++ b/packages/tiers/src/utils/mainResponse.ts @@ -6,9 +6,9 @@ type NavigationResponse = MinimalResponse & Pick // A redirect loop is capped by the browser long before this, but the chain is caller- // visible data so it gets a bound of its own. -const MAX_REDIRECT_ENTRIES = captureLimit(process.env.CAPTURE_MAX_REDIRECT_ENTRIES, 50) -const MAX_REDIRECT_STRING_CHARS = captureLimit(process.env.CAPTURE_MAX_STRING_CHARS, 2_000) -const MAX_REDIRECT_TOTAL_CHARS = captureLimit(process.env.CAPTURE_MAX_TOTAL_CHARS, 1_000_000) +const MAX_REDIRECT_ENTRIES = captureLimit(process.env.REDIRECT_MAX_ENTRIES, 50) +const MAX_REDIRECT_URL_CHARS = captureLimit(process.env.REDIRECT_MAX_URL_CHARS, 2_000) +const MAX_REDIRECT_TOTAL_CHARS = captureLimit(process.env.REDIRECT_MAX_TOTAL_CHARS, 1_000_000) /** Tracks the latest top-level document response across redirects. */ export class MainDocumentResponseTracker { @@ -35,7 +35,7 @@ export class MainDocumentResponseTracker { private record(url: string): void { if ( this.chain.length >= MAX_REDIRECT_ENTRIES || - url.length > MAX_REDIRECT_STRING_CHARS || + url.length > MAX_REDIRECT_URL_CHARS || this.chainChars + url.length > MAX_REDIRECT_TOTAL_CHARS || this.chain.includes(url) ) diff --git a/packages/tiers/src/utils/responseCapture.ts b/packages/tiers/src/utils/responseCapture.ts index cd11a96..4042a58 100644 --- a/packages/tiers/src/utils/responseCapture.ts +++ b/packages/tiers/src/utils/responseCapture.ts @@ -3,16 +3,16 @@ import type { Page, Response } from "patchright" import { captureLimit } from "./captureConfig" import { isTextContentType } from "./response" -const MAX_PATTERNS = captureLimit(process.env.CAPTURE_MAX_RESPONSE_PATTERNS, 10) +const MAX_PATTERNS = captureLimit(process.env.CAPTURE_MAX_PATTERNS, 10) const MAX_RESPONSES = captureLimit(process.env.CAPTURE_MAX_RESPONSES, 5) -const MAX_BODY_BYTES = captureLimit(process.env.CAPTURE_MAX_RESPONSE_BYTES, 5_242_880) -const MAX_TOTAL_BYTES = captureLimit(process.env.CAPTURE_MAX_RESPONSE_TOTAL_BYTES, 10_485_760) +const MAX_BODY_BYTES = captureLimit(process.env.CAPTURE_MAX_BODY_BYTES, 5_242_880) +const MAX_TOTAL_BYTES = captureLimit(process.env.CAPTURE_MAX_TOTAL_BYTES, 10_485_760) const MAX_READ_BYTES = captureLimit(process.env.CAPTURE_MAX_READ_BYTES, 10_485_760) const BODY_TIMEOUT_MS = captureLimit(process.env.CAPTURE_BODY_TIMEOUT_MS, 5_000) const SETTLE_MS = captureLimit(process.env.CAPTURE_SETTLE_MS, 15_000) const MAX_SETTLE_MS = captureLimit(process.env.CAPTURE_MAX_SETTLE_MS, 60_000) -const IDLE_FLOOR_MS = captureLimit(process.env.CAPTURE_SETTLE_IDLE_FLOOR_MS, 5_000) -const MAX_STRING_CHARS = captureLimit(process.env.CAPTURE_MAX_STRING_CHARS, 2_000) +const IDLE_FLOOR_MS = captureLimit(process.env.CAPTURE_IDLE_FLOOR_MS, 5_000) +const MAX_STRING_CHARS = captureLimit(process.env.CAPTURE_MAX_METADATA_CHARS, 2_000) const NEVER = new Promise(() => {}) const COMPRESSED_ENCODINGS = new Set(["gzip", "br", "deflate", "zstd"])