diff --git a/.env.example b/.env.example index 7d389f7..f892661 100644 --- a/.env.example +++ b/.env.example @@ -55,6 +55,10 @@ CAPTURE_SETTLE_IDLE_FLOOR_MS=5000 # Sessions # Redis stores solved cookies and browser sessions. REDIS_URL=redis://localhost:6379 +# Maximum time for each Redis connection attempt. +REDIS_CONNECT_TIMEOUT_MS=5000 +# Delay between background reconnect attempts. Set 0 to disable retry. +REDIS_RETRY_DELAY_MS=5000 # Lifetime of cached sessions in seconds. SESSION_TTL_SECONDS=3600 diff --git a/CHANGELOG.md b/CHANGELOG.md index 63a8a3f..e8a5445 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Optional viewport screenshot: `screenshot: true` on `POST /scrape` returns a base64 JPEG of the viewport in `ScrapeResult.screenshot`, captured by the browser tiers (2-4) immediately before the HTML read so image and markup describe the same moment. Off by default; a stock request attaches nothing and does no extra work. Settle wait, capture timeout, JPEG quality, and maximum image size are bounded and tunable via `SCREENSHOT_*`, and a capture failure leaves the field unset rather than failing the scrape. ### Fixed +- Recover the Redis-backed Tier 2 cache after a transient startup timeout without restarting TRAWL. Connection attempts are bounded by `REDIS_CONNECT_TIMEOUT_MS`, retry in the background after `REDIS_RETRY_DELAY_MS`, and stop cleanly during shutdown; setting the retry delay to `0` disables reconnects for intentionally cacheless deployments (#92). - Correct Docker troubleshooting commands to use the actual `trawl` Compose service name, and distinguish Prowlarr's always-available FlareSolverr API on port 8191 from the opt-in forward proxy on port 8192 (#96). - Wait for the bundled Redis service to pass a `PING` healthcheck before starting TRAWL, preventing a transient Compose startup race from disabling the Tier 2 session cache for the process lifetime (#90). - Reap orphaned Camoufox processes in both API container variants by running Bun under Tini (#79). diff --git a/README.md b/README.md index a80f341..c1e6c00 100644 --- a/README.md +++ b/README.md @@ -396,6 +396,8 @@ for pool and mounted-file examples. | `BROWSER_CONTENT_PROCESSES` | `2` | Cap Firefox content processes per browser (`dom.ipc.processCount`); lowers RAM/CPU | | `SESSION_TTL_SECONDS` | `3600` | Redis session cache TTL (seconds) | | `REDIS_URL` | `redis://localhost:6379` | Redis connection string | +| `REDIS_CONNECT_TIMEOUT_MS` | `5000` | Maximum time for each Redis connection attempt | +| `REDIS_RETRY_DELAY_MS` | `5000` | Delay before reconnecting after startup failure; `0` disables retry | | `PROXY_URL` | — | Optional Tier 3 HTTP or SOCKS5 proxy, or comma-separated pool | | `PROXY_LIST_FILE` | — | File containing one Tier 3 proxy URL per line | | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation | diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index 8457755..180d26f 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -8,6 +8,18 @@ export const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3") // Tune lower for fast-fail feedback in dev; tune higher for very heavy upstream targets. export const ACQUIRE_TIMEOUT_MS = Number(process.env.BROWSER_ACQUIRE_TIMEOUT_MS ?? "15000") export const SESSION_TTL = Number(process.env.SESSION_TTL_SECONDS ?? "3600") +const positiveInteger = (value: string | undefined, fallback: number): number => { + const parsed = Number(value) + return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback +} +const nonNegativeInteger = (value: string | undefined, fallback: number): number => { + const parsed = Number(value) + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : fallback +} +// A failed initial Redis connection must not disable Tier 2 for the process lifetime. +// Each attempt is bounded; failed attempts are retried in the background while the API stays ready. +export const REDIS_CONNECT_TIMEOUT_MS = positiveInteger(process.env.REDIS_CONNECT_TIMEOUT_MS, 5_000) +export const REDIS_RETRY_DELAY_MS = nonNegativeInteger(process.env.REDIS_RETRY_DELAY_MS, 5_000) // Rolling-replace a browser after this many Tier 3/4 temporary contexts. Every // creation counts regardless of outcome; 0 disables periodic replacement. export const RECYCLE_AFTER_TEMPORARY_CONTEXTS = Number(process.env.BROWSER_RECYCLE_AFTER_CONTEXTS ?? "8") diff --git a/apps/api/src/deps.test.ts b/apps/api/src/deps.test.ts index 973d911..3d6244f 100644 --- a/apps/api/src/deps.test.ts +++ b/apps/api/src/deps.test.ts @@ -1,7 +1,9 @@ import { describe, expect, test } from "bun:test" import type { BrowserPool } from "@trawl/browser" import type { BrowserHandle } from "@trawl/types" -import { getDeps, getHeadfulPool, initPool, shutdownPools } from "./deps" +import { getDeps, getHeadfulPool, initPool, SessionCacheRecovery, shutdownPools } from "./deps" + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) const handle = (headful: boolean): BrowserHandle => ({ id: 0, @@ -80,3 +82,89 @@ describe("browser pool dependencies", () => { expect(factory.pools[1]?.events).toEqual(["init", "shutdown"]) }) }) + +describe("session cache recovery", () => { + test("enables the cache after a failed initial connection without restarting", async () => { + let attempts = 0 + const closed: number[] = [] + const connected: number[] = [] + const recovery = new SessionCacheRecovery({ + createCache: () => { + const id = ++attempts + return { + connect: async () => { + if (id === 1) throw new Error("redis still starting") + }, + close: () => closed.push(id), + load: async () => undefined, + save: async () => {}, + invalidate: async () => {}, + } + }, + connectTimeoutMs: 10, + retryDelayMs: 5, + onConnected: () => connected.push(attempts), + }) + + await recovery.start() + expect(recovery.current()).toBeUndefined() + await sleep(15) + + expect(attempts).toBe(2) + expect(closed).toEqual([1]) + expect(connected).toEqual([2]) + expect(recovery.current()).toBeDefined() + + await recovery.stop() + expect(closed).toEqual([1, 2]) + }) + + test("cancels a pending retry during shutdown", async () => { + let attempts = 0 + const recovery = new SessionCacheRecovery({ + createCache: () => ({ + connect: async () => { + attempts++ + throw new Error("offline") + }, + close: () => {}, + load: async () => undefined, + save: async () => {}, + invalidate: async () => {}, + }), + connectTimeoutMs: 10, + retryDelayMs: 10, + }) + + await recovery.start() + await recovery.stop() + await sleep(20) + + expect(attempts).toBe(1) + expect(recovery.current()).toBeUndefined() + }) + + test("supports intentionally cacheless deployments without a retry loop", async () => { + let attempts = 0 + const recovery = new SessionCacheRecovery({ + createCache: () => ({ + connect: async () => { + attempts++ + throw new Error("disabled") + }, + close: () => {}, + load: async () => undefined, + save: async () => {}, + invalidate: async () => {}, + }), + connectTimeoutMs: 10, + retryDelayMs: 0, + }) + + await recovery.start() + await sleep(15) + + expect(attempts).toBe(1) + await recovery.stop() + }) +}) diff --git a/apps/api/src/deps.ts b/apps/api/src/deps.ts index 1aa11ee..a1e2ed0 100644 --- a/apps/api/src/deps.ts +++ b/apps/api/src/deps.ts @@ -10,6 +10,8 @@ import { POOL_SIZE, proxyPool, RECYCLE_AFTER_TEMPORARY_CONTEXTS, + REDIS_CONNECT_TIMEOUT_MS, + REDIS_RETRY_DELAY_MS, REDIS_URL, residentialProxyPool, SESSION_TTL, @@ -19,13 +21,89 @@ import { const state: { pool?: BrowserPool headfulPool?: BrowserPool - sessionCache?: SessionCache } = {} const handleOwners = new WeakMap() type BrowserPoolOptions = ConstructorParameters[0] +interface SessionCacheClient { + connect(timeoutMs?: number): Promise + close(): void + load(domain: string): Promise + save(domain: string, data: SessionData): Promise + invalidate(domain: string): Promise +} + +interface SessionCacheRecoveryOptions { + createCache: () => SessionCacheClient + connectTimeoutMs: number + retryDelayMs: number + onConnected?: () => void + onUnavailable?: (error: unknown) => void +} + +export class SessionCacheRecovery { + private cache?: SessionCacheClient + private retryTimer?: ReturnType + private stopped = true + + constructor(private readonly options: SessionCacheRecoveryOptions) {} + + current(): SessionCacheClient | undefined { + return this.cache + } + + async start(): Promise { + await this.stop() + this.stopped = false + await this.connect() + } + + async stop(): Promise { + this.stopped = true + if (this.retryTimer) clearTimeout(this.retryTimer) + this.retryTimer = undefined + this.cache?.close() + this.cache = undefined + } + + private async connect(): Promise { + if (this.stopped) return + const candidate = this.options.createCache() + try { + await candidate.connect(this.options.connectTimeoutMs) + if (this.stopped) { + candidate.close() + return + } + this.cache = candidate + this.options.onConnected?.() + } catch (error) { + candidate.close() + if (this.stopped) return + this.options.onUnavailable?.(error) + if (this.options.retryDelayMs === 0) return + this.retryTimer = setTimeout(() => { + this.retryTimer = undefined + void this.connect() + }, this.options.retryDelayMs) + this.retryTimer.unref?.() + } + } +} + +const sessionCacheRecovery = new SessionCacheRecovery({ + createCache: () => new SessionCache({ redisUrl: REDIS_URL, ttlSeconds: SESSION_TTL }), + connectTimeoutMs: REDIS_CONNECT_TIMEOUT_MS, + retryDelayMs: REDIS_RETRY_DELAY_MS, + onConnected: () => console.log("[api] session cache connected (Tier 2 fast-path enabled)"), + onUnavailable: (err) => { + const retry = REDIS_RETRY_DELAY_MS > 0 ? `; retrying in ${REDIS_RETRY_DELAY_MS}ms` : "" + console.warn(`[api] session cache unavailable — Tier 2 disabled${retry}:`, err instanceof Error ? err.message : err) + }, +}) + interface InitPoolOptions { poolSize?: number headfulPoolSize?: number @@ -36,23 +114,10 @@ interface InitPoolOptions { export const getPool = () => state.pool export const getHeadfulPool = () => state.headfulPool -const initSessionCache = async (): Promise => { - try { - const sessionCache = new SessionCache({ - redisUrl: REDIS_URL, - ttlSeconds: SESSION_TTL, - }) - await sessionCache.connect() - state.sessionCache = sessionCache - console.log("[api] session cache connected (Tier 2 fast-path enabled)") - } catch (err) { - state.sessionCache = undefined - console.warn("[api] session cache unavailable — Tier 2 disabled:", err instanceof Error ? err.message : err) - } -} +const initSessionCache = (): Promise => sessionCacheRecovery.start() export const shutdownPools = async (): Promise => { - await Promise.all([state.pool?.shutdown(), state.headfulPool?.shutdown()]) + await Promise.all([sessionCacheRecovery.stop(), state.pool?.shutdown(), state.headfulPool?.shutdown()]) } export const initPool = async ({ @@ -126,11 +191,20 @@ export const getDeps = (): OrchestratorDeps => { handleOwners.delete(handle) }, loadSession: (d: string) => - state.sessionCache ? state.sessionCache.load(d).catch(() => undefined) : Promise.resolve(undefined), + sessionCacheRecovery + .current() + ?.load(d) + .catch(() => undefined) ?? Promise.resolve(undefined), saveSession: (d: string, data: SessionData) => - state.sessionCache ? state.sessionCache.save(d, data).catch(() => {}) : Promise.resolve(), + sessionCacheRecovery + .current() + ?.save(d, data) + .catch(() => {}) ?? Promise.resolve(), invalidateSession: (d: string) => - state.sessionCache ? state.sessionCache.invalidate(d).catch(() => {}) : Promise.resolve(), + sessionCacheRecovery + .current() + ?.invalidate(d) + .catch(() => {}) ?? Promise.resolve(), proxyPool, residentialProxyPool, } diff --git a/apps/docs/architecture/session-cache.md b/apps/docs/architecture/session-cache.md index e851d43..21a017a 100644 --- a/apps/docs/architecture/session-cache.md +++ b/apps/docs/architecture/session-cache.md @@ -63,6 +63,11 @@ This handles provider cookies expiring or being rejected before the Redis TTL en TRAWL's cache backend is Redis 8.8. TRAWL talks to it with `new RedisClient(REDIS_URL)` from Bun's native Redis client (not ioredis). +Each connection attempt is bounded by `REDIS_CONNECT_TIMEOUT_MS` (default 5 seconds). If Redis is +not ready, scraping continues without Tier 2 while TRAWL retries in the background every +`REDIS_RETRY_DELAY_MS` (default 5 seconds). Set the retry delay to `0` when Redis is intentionally +absent. + ```typescript import { RedisClient } from 'bun' diff --git a/apps/docs/deployment/docker-compose.md b/apps/docs/deployment/docker-compose.md index 03ea596..c2c0654 100644 --- a/apps/docs/deployment/docker-compose.md +++ b/apps/docs/deployment/docker-compose.md @@ -119,6 +119,8 @@ TRAWL and Redis. | `BROWSER_ACQUIRE_TIMEOUT_MS` | `15000` | How long `acquire()` polls for a free browser before returning HTTP 429 | | `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8` | Rolling-replace after this many Tier 3/4 contexts; `0` disables it | | `REDIS_URL` | `redis://redis:6379` | Redis connection (set automatically in compose) | +| `REDIS_CONNECT_TIMEOUT_MS` | `5000` | Maximum time for each Redis connection attempt | +| `REDIS_RETRY_DELAY_MS` | `5000` | Background reconnect delay; `0` disables retry | | `PROXY_URL` | — | Optional Tier 3 datacenter proxy or pool | | `RESIDENTIAL_PROXY_URL` | — | Enables Tier 4 proxy escalation | | `MITM_PROXY_ENABLED` | `false` | Starts the general HTTP/HTTPS proxy | diff --git a/apps/docs/deployment/troubleshooting.md b/apps/docs/deployment/troubleshooting.md index 2cceab8..ee2ce69 100644 --- a/apps/docs/deployment/troubleshooting.md +++ b/apps/docs/deployment/troubleshooting.md @@ -17,8 +17,9 @@ description: Common issues and how to fix them. `docker compose pull trawl && docker compose up -d --force-recreate trawl`. 2. **shm_size too small** — Ensure `shm_size: 1gb` is set on the API service. -Redis is optional at runtime. If it is unavailable, TRAWL disables the Tier 2 session-cache -fast path but can still become ready and scrape through the other tiers. +Redis is optional at runtime. If it is unavailable, TRAWL temporarily disables the Tier 2 +session-cache fast path but can still become ready and scrape through the other tiers. Unless +`REDIS_RETRY_DELAY_MS=0`, it keeps reconnecting in the background. ## Logs report `Tier 2 disabled` @@ -33,13 +34,11 @@ docker compose exec trawl sh -lc 'printf "%s\n" "$REDIS_URL"; getent hosts redis docker compose exec trawl bun -e 'import { RedisClient } from "bun"; const client = new RedisClient(process.env.REDIS_URL); await client.connect(); console.log(await client.ping()); client.close()' ``` -With the supplied Compose files, TRAWL waits for the Redis healthcheck before starting. If the -second command returns `PONG` on a deployment that logged the warning during an earlier start, -restart only TRAWL to enable Tier 2: - -```bash -docker compose restart trawl -``` +With the supplied Redis-backed Compose files, TRAWL waits for the Redis healthcheck before +starting. A slow host may still exceed the client-side connection timeout; TRAWL now retries in the +background and logs `session cache connected` when Tier 2 becomes available, without a process +restart. Increase `REDIS_CONNECT_TIMEOUT_MS` if every attempt times out, or adjust +`REDIS_RETRY_DELAY_MS` to change the retry interval. If the command fails, inspect `docker compose config` for an overridden `REDIS_URL`, custom `network_mode`, or networks that are not shared by the `trawl` and `redis` services. Inside Docker, diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index 2f20c39..0cb3f58 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -57,6 +57,20 @@ With a specific database index: REDIS_URL=redis://redis:6379/1 ``` +### `REDIS_CONNECT_TIMEOUT_MS` + +**Default:** `5000` + +Maximum duration of each Redis connection attempt. A failed attempt does not block the API or +permanently disable Tier 2; TRAWL continues without the cache and reconnects in the background. + +### `REDIS_RETRY_DELAY_MS` + +**Default:** `5000` + +Delay between background connection attempts. Set it to `0` when Redis is intentionally absent to +disable retries. The supplied minimal Compose variant does this automatically. + ## Browser Pool ### `BROWSER_POOL_SIZE` diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index bd2d44c..ce9f6ca 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -7,6 +7,8 @@ services: - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}" shm_size: 1gb environment: + # This variant has no Redis service, so do not keep retrying after the initial cache probe. + REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-0} BROWSER_POOL_SIZE: 1 BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0} MCP_ENABLED: ${MCP_ENABLED:-false} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 400e4b8..7bf67f6 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -22,6 +22,8 @@ services: mem_limit: 3g environment: REDIS_URL: redis://redis:6379 + REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-5000} + REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-5000} BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-3} BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0} MCP_ENABLED: ${MCP_ENABLED:-false} diff --git a/docker-compose.yml b/docker-compose.yml index 4d0e02f..7e59689 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,8 @@ services: shm_size: 1gb environment: REDIS_URL: redis://redis:6379 + REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-5000} + REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-5000} BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-1} BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0} MCP_ENABLED: ${MCP_ENABLED:-false} diff --git a/packages/browser/src/session.ts b/packages/browser/src/session.ts index e48c83e..d89a490 100644 --- a/packages/browser/src/session.ts +++ b/packages/browser/src/session.ts @@ -27,6 +27,10 @@ export class SessionCache { } } + close(): void { + this.redis.close() + } + private key(domain: string): string { return `session:${domain}` }