fix(cache): reconnect Redis after startup timeout

.env.example

@@ -55,6 +55,10 @@ CAPTURE_SETTLE_IDLE_FLOOR_MS=5000
 # Sessions
 # Redis stores solved cookies and browser sessions.
 REDIS_URL=redis://localhost:6379
+# Maximum time for each Redis connection attempt.
+REDIS_CONNECT_TIMEOUT_MS=5000
+# Delay between background reconnect attempts. Set 0 to disable retry.
+REDIS_RETRY_DELAY_MS=5000
 # Lifetime of cached sessions in seconds.
 SESSION_TTL_SECONDS=3600
 

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Optional viewport screenshot: `screenshot: true` on `POST /scrape` returns a base64 JPEG of the viewport in `ScrapeResult.screenshot`, captured by the browser tiers (2-4) immediately before the HTML read so image and markup describe the same moment. Off by default; a stock request attaches nothing and does no extra work. Settle wait, capture timeout, JPEG quality, and maximum image size are bounded and tunable via `SCREENSHOT_*`, and a capture failure leaves the field unset rather than failing the scrape.
 
 ### Fixed
+- Recover the Redis-backed Tier 2 cache after a transient startup timeout without restarting TRAWL. Connection attempts are bounded by `REDIS_CONNECT_TIMEOUT_MS`, retry in the background after `REDIS_RETRY_DELAY_MS`, and stop cleanly during shutdown; setting the retry delay to `0` disables reconnects for intentionally cacheless deployments (#92).
 - Correct Docker troubleshooting commands to use the actual `trawl` Compose service name, and distinguish Prowlarr's always-available FlareSolverr API on port 8191 from the opt-in forward proxy on port 8192 (#96).
 - Wait for the bundled Redis service to pass a `PING` healthcheck before starting TRAWL, preventing a transient Compose startup race from disabling the Tier 2 session cache for the process lifetime (#90).
 - Reap orphaned Camoufox processes in both API container variants by running Bun under Tini (#79).

README.md

@@ -396,6 +396,8 @@ for pool and mounted-file examples.
 | `BROWSER_CONTENT_PROCESSES`      | `2`                      | Cap Firefox content processes per browser (`dom.ipc.processCount`); lowers RAM/CPU  |
 | `SESSION_TTL_SECONDS`            | `3600`                   | Redis session cache TTL (seconds)                                                   |
 | `REDIS_URL`                      | `redis://localhost:6379` | Redis connection string                                                             |
+| `REDIS_CONNECT_TIMEOUT_MS`       | `5000`                   | Maximum time for each Redis connection attempt                                      |
+| `REDIS_RETRY_DELAY_MS`           | `5000`                   | Delay before reconnecting after startup failure; `0` disables retry                 |
 | `PROXY_URL`                      | —                        | Optional Tier 3 HTTP or SOCKS5 proxy, or comma-separated pool                       |
 | `PROXY_LIST_FILE`                | —                        | File containing one Tier 3 proxy URL per line                                       |
 | `RESIDENTIAL_PROXY_URL`          | —                        | Enables Tier 4 proxy escalation                                                     |

apps/api/src/config.ts

@@ -8,6 +8,18 @@ export const POOL_SIZE = Number(process.env.BROWSER_POOL_SIZE ?? "3")
 // Tune lower for fast-fail feedback in dev; tune higher for very heavy upstream targets.
 export const ACQUIRE_TIMEOUT_MS = Number(process.env.BROWSER_ACQUIRE_TIMEOUT_MS ?? "15000")
 export const SESSION_TTL = Number(process.env.SESSION_TTL_SECONDS ?? "3600")
+const positiveInteger = (value: string | undefined, fallback: number): number => {
+  const parsed = Number(value)
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback
+}
+const nonNegativeInteger = (value: string | undefined, fallback: number): number => {
+  const parsed = Number(value)
+  return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : fallback
+}
+// A failed initial Redis connection must not disable Tier 2 for the process lifetime.
+// Each attempt is bounded; failed attempts are retried in the background while the API stays ready.
+export const REDIS_CONNECT_TIMEOUT_MS = positiveInteger(process.env.REDIS_CONNECT_TIMEOUT_MS, 5_000)
+export const REDIS_RETRY_DELAY_MS = nonNegativeInteger(process.env.REDIS_RETRY_DELAY_MS, 5_000)
 // Rolling-replace a browser after this many Tier 3/4 temporary contexts. Every
 // creation counts regardless of outcome; 0 disables periodic replacement.
 export const RECYCLE_AFTER_TEMPORARY_CONTEXTS = Number(process.env.BROWSER_RECYCLE_AFTER_CONTEXTS ?? "8")

apps/api/src/deps.test.ts

@@ -1,7 +1,9 @@
 import { describe, expect, test } from "bun:test"
 import type { BrowserPool } from "@trawl/browser"
 import type { BrowserHandle } from "@trawl/types"
-import { getDeps, getHeadfulPool, initPool, shutdownPools } from "./deps"
+import { getDeps, getHeadfulPool, initPool, SessionCacheRecovery, shutdownPools } from "./deps"
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
 
 const handle = (headful: boolean): BrowserHandle => ({
   id: 0,
@@ -80,3 +82,89 @@ describe("browser pool dependencies", () => {
     expect(factory.pools[1]?.events).toEqual(["init", "shutdown"])
   })
 })
+
+describe("session cache recovery", () => {
+  test("enables the cache after a failed initial connection without restarting", async () => {
+    let attempts = 0
+    const closed: number[] = []
+    const connected: number[] = []
+    const recovery = new SessionCacheRecovery({
+      createCache: () => {
+        const id = ++attempts
+        return {
+          connect: async () => {
+            if (id === 1) throw new Error("redis still starting")
+          },
+          close: () => closed.push(id),
+          load: async () => undefined,
+          save: async () => {},
+          invalidate: async () => {},
+        }
+      },
+      connectTimeoutMs: 10,
+      retryDelayMs: 5,
+      onConnected: () => connected.push(attempts),
+    })
+
+    await recovery.start()
+    expect(recovery.current()).toBeUndefined()
+    await sleep(15)
+
+    expect(attempts).toBe(2)
+    expect(closed).toEqual([1])
+    expect(connected).toEqual([2])
+    expect(recovery.current()).toBeDefined()
+
+    await recovery.stop()
+    expect(closed).toEqual([1, 2])
+  })
+
+  test("cancels a pending retry during shutdown", async () => {
+    let attempts = 0
+    const recovery = new SessionCacheRecovery({
+      createCache: () => ({
+        connect: async () => {
+          attempts++
+          throw new Error("offline")
+        },
+        close: () => {},
+        load: async () => undefined,
+        save: async () => {},
+        invalidate: async () => {},
+      }),
+      connectTimeoutMs: 10,
+      retryDelayMs: 10,
+    })
+
+    await recovery.start()
+    await recovery.stop()
+    await sleep(20)
+
+    expect(attempts).toBe(1)
+    expect(recovery.current()).toBeUndefined()
+  })
+
+  test("supports intentionally cacheless deployments without a retry loop", async () => {
+    let attempts = 0
+    const recovery = new SessionCacheRecovery({
+      createCache: () => ({
+        connect: async () => {
+          attempts++
+          throw new Error("disabled")
+        },
+        close: () => {},
+        load: async () => undefined,
+        save: async () => {},
+        invalidate: async () => {},
+      }),
+      connectTimeoutMs: 10,
+      retryDelayMs: 0,
+    })
+
+    await recovery.start()
+    await sleep(15)
+
+    expect(attempts).toBe(1)
+    await recovery.stop()
+  })
+})

apps/api/src/deps.ts

@@ -10,6 +10,8 @@ import {
   POOL_SIZE,
   proxyPool,
   RECYCLE_AFTER_TEMPORARY_CONTEXTS,
+  REDIS_CONNECT_TIMEOUT_MS,
+  REDIS_RETRY_DELAY_MS,
   REDIS_URL,
   residentialProxyPool,
   SESSION_TTL,
@@ -19,13 +21,89 @@ import {
 const state: {
   pool?: BrowserPool
   headfulPool?: BrowserPool
-  sessionCache?: SessionCache
 } = {}
 
 const handleOwners = new WeakMap<object, BrowserPool>()
 
 type BrowserPoolOptions = ConstructorParameters<typeof BrowserPool>[0]
 
+interface SessionCacheClient {
+  connect(timeoutMs?: number): Promise<void>
+  close(): void
+  load(domain: string): Promise<SessionData | undefined>
+  save(domain: string, data: SessionData): Promise<void>
+  invalidate(domain: string): Promise<void>
+}
+
+interface SessionCacheRecoveryOptions {
+  createCache: () => SessionCacheClient
+  connectTimeoutMs: number
+  retryDelayMs: number
+  onConnected?: () => void
+  onUnavailable?: (error: unknown) => void
+}
+
+export class SessionCacheRecovery {
+  private cache?: SessionCacheClient
+  private retryTimer?: ReturnType<typeof setTimeout>
+  private stopped = true
+
+  constructor(private readonly options: SessionCacheRecoveryOptions) {}
+
+  current(): SessionCacheClient | undefined {
+    return this.cache
+  }
+
+  async start(): Promise<void> {
+    await this.stop()
+    this.stopped = false
+    await this.connect()
+  }
+
+  async stop(): Promise<void> {
+    this.stopped = true
+    if (this.retryTimer) clearTimeout(this.retryTimer)
+    this.retryTimer = undefined
+    this.cache?.close()
+    this.cache = undefined
+  }
+
+  private async connect(): Promise<void> {
+    if (this.stopped) return
+    const candidate = this.options.createCache()
+    try {
+      await candidate.connect(this.options.connectTimeoutMs)
+      if (this.stopped) {
+        candidate.close()
+        return
+      }
+      this.cache = candidate
+      this.options.onConnected?.()
+    } catch (error) {
+      candidate.close()
+      if (this.stopped) return
+      this.options.onUnavailable?.(error)
+      if (this.options.retryDelayMs === 0) return
+      this.retryTimer = setTimeout(() => {
+        this.retryTimer = undefined
+        void this.connect()
+      }, this.options.retryDelayMs)
+      this.retryTimer.unref?.()
+    }
+  }
+}
+
+const sessionCacheRecovery = new SessionCacheRecovery({
+  createCache: () => new SessionCache({ redisUrl: REDIS_URL, ttlSeconds: SESSION_TTL }),
+  connectTimeoutMs: REDIS_CONNECT_TIMEOUT_MS,
+  retryDelayMs: REDIS_RETRY_DELAY_MS,
+  onConnected: () => console.log("[api] session cache connected  (Tier 2 fast-path enabled)"),
+  onUnavailable: (err) => {
+    const retry = REDIS_RETRY_DELAY_MS > 0 ? `; retrying in ${REDIS_RETRY_DELAY_MS}ms` : ""
+    console.warn(`[api] session cache unavailable — Tier 2 disabled${retry}:`, err instanceof Error ? err.message : err)
+  },
+})
+
 interface InitPoolOptions {
   poolSize?: number
   headfulPoolSize?: number
@@ -36,23 +114,10 @@ interface InitPoolOptions {
 export const getPool = () => state.pool
 export const getHeadfulPool = () => state.headfulPool
 
-const initSessionCache = async (): Promise<void> => {
-  try {
-    const sessionCache = new SessionCache({
-      redisUrl: REDIS_URL,
-      ttlSeconds: SESSION_TTL,
-    })
-    await sessionCache.connect()
-    state.sessionCache = sessionCache
-    console.log("[api] session cache connected  (Tier 2 fast-path enabled)")
-  } catch (err) {
-    state.sessionCache = undefined
-    console.warn("[api] session cache unavailable — Tier 2 disabled:", err instanceof Error ? err.message : err)
-  }
-}
+const initSessionCache = (): Promise<void> => sessionCacheRecovery.start()
 
 export const shutdownPools = async (): Promise<void> => {
-  await Promise.all([state.pool?.shutdown(), state.headfulPool?.shutdown()])
+  await Promise.all([sessionCacheRecovery.stop(), state.pool?.shutdown(), state.headfulPool?.shutdown()])
 }
 
 export const initPool = async ({
@@ -126,11 +191,20 @@ export const getDeps = (): OrchestratorDeps => {
       handleOwners.delete(handle)
     },
     loadSession: (d: string) =>
-      state.sessionCache ? state.sessionCache.load(d).catch(() => undefined) : Promise.resolve(undefined),
+      sessionCacheRecovery
+        .current()
+        ?.load(d)
+        .catch(() => undefined) ?? Promise.resolve(undefined),
     saveSession: (d: string, data: SessionData) =>
-      state.sessionCache ? state.sessionCache.save(d, data).catch(() => {}) : Promise.resolve(),
+      sessionCacheRecovery
+        .current()
+        ?.save(d, data)
+        .catch(() => {}) ?? Promise.resolve(),
     invalidateSession: (d: string) =>
-      state.sessionCache ? state.sessionCache.invalidate(d).catch(() => {}) : Promise.resolve(),
+      sessionCacheRecovery
+        .current()
+        ?.invalidate(d)
+        .catch(() => {}) ?? Promise.resolve(),
     proxyPool,
     residentialProxyPool,
   }

apps/docs/architecture/session-cache.md

@@ -63,6 +63,11 @@ This handles provider cookies expiring or being rejected before the Redis TTL en
 
 TRAWL's cache backend is Redis 8.8. TRAWL talks to it with `new RedisClient(REDIS_URL)` from Bun's native Redis client (not ioredis).
 
+Each connection attempt is bounded by `REDIS_CONNECT_TIMEOUT_MS` (default 5 seconds). If Redis is
+not ready, scraping continues without Tier 2 while TRAWL retries in the background every
+`REDIS_RETRY_DELAY_MS` (default 5 seconds). Set the retry delay to `0` when Redis is intentionally
+absent.
+
 ```typescript
 import { RedisClient } from 'bun'
 

apps/docs/deployment/docker-compose.md

@@ -119,6 +119,8 @@ TRAWL and Redis.
 | `BROWSER_ACQUIRE_TIMEOUT_MS`     | `15000`              | How long `acquire()` polls for a free browser before returning HTTP 429 |
 | `BROWSER_RECYCLE_AFTER_CONTEXTS` | `8`                  | Rolling-replace after this many Tier 3/4 contexts; `0` disables it      |
 | `REDIS_URL`                      | `redis://redis:6379` | Redis connection (set automatically in compose)                         |
+| `REDIS_CONNECT_TIMEOUT_MS`       | `5000`               | Maximum time for each Redis connection attempt                          |
+| `REDIS_RETRY_DELAY_MS`           | `5000`               | Background reconnect delay; `0` disables retry                          |
 | `PROXY_URL`                      | —                    | Optional Tier 3 datacenter proxy or pool                                |
 | `RESIDENTIAL_PROXY_URL`          | —                    | Enables Tier 4 proxy escalation                                         |
 | `MITM_PROXY_ENABLED`             | `false`              | Starts the general HTTP/HTTPS proxy                                     |

apps/docs/deployment/troubleshooting.md

@@ -17,8 +17,9 @@ description: Common issues and how to fix them.
    `docker compose pull trawl && docker compose up -d --force-recreate trawl`.
 2. **shm_size too small** — Ensure `shm_size: 1gb` is set on the API service.
 
-Redis is optional at runtime. If it is unavailable, TRAWL disables the Tier 2 session-cache
-fast path but can still become ready and scrape through the other tiers.
+Redis is optional at runtime. If it is unavailable, TRAWL temporarily disables the Tier 2
+session-cache fast path but can still become ready and scrape through the other tiers. Unless
+`REDIS_RETRY_DELAY_MS=0`, it keeps reconnecting in the background.
 
 ## Logs report `Tier 2 disabled`
 
@@ -33,13 +34,11 @@ docker compose exec trawl sh -lc 'printf "%s\n" "$REDIS_URL"; getent hosts redis
 docker compose exec trawl bun -e 'import { RedisClient } from "bun"; const client = new RedisClient(process.env.REDIS_URL); await client.connect(); console.log(await client.ping()); client.close()'
 ```
 
-With the supplied Compose files, TRAWL waits for the Redis healthcheck before starting. If the
-second command returns `PONG` on a deployment that logged the warning during an earlier start,
-restart only TRAWL to enable Tier 2:
-
-```bash
-docker compose restart trawl
-```
+With the supplied Redis-backed Compose files, TRAWL waits for the Redis healthcheck before
+starting. A slow host may still exceed the client-side connection timeout; TRAWL now retries in the
+background and logs `session cache connected` when Tier 2 becomes available, without a process
+restart. Increase `REDIS_CONNECT_TIMEOUT_MS` if every attempt times out, or adjust
+`REDIS_RETRY_DELAY_MS` to change the retry interval.
 
 If the command fails, inspect `docker compose config` for an overridden `REDIS_URL`, custom
 `network_mode`, or networks that are not shared by the `trawl` and `redis` services. Inside Docker,

apps/docs/getting-started/configuration.md

@@ -57,6 +57,20 @@ With a specific database index:
 REDIS_URL=redis://redis:6379/1
 ```
 
+### `REDIS_CONNECT_TIMEOUT_MS`
+
+**Default:** `5000`
+
+Maximum duration of each Redis connection attempt. A failed attempt does not block the API or
+permanently disable Tier 2; TRAWL continues without the cache and reconnects in the background.
+
+### `REDIS_RETRY_DELAY_MS`
+
+**Default:** `5000`
+
+Delay between background connection attempts. Set it to `0` when Redis is intentionally absent to
+disable retries. The supplied minimal Compose variant does this automatically.
+
 ## Browser Pool
 
 ### `BROWSER_POOL_SIZE`

docker-compose.minimal.yml

@@ -7,6 +7,8 @@ services:
       - "${MITM_PROXY_PORT:-8192}:${MITM_PROXY_PORT:-8192}"
     shm_size: 1gb
     environment:
+      # This variant has no Redis service, so do not keep retrying after the initial cache probe.
+      REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-0}
       BROWSER_POOL_SIZE: 1
       BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0}
       MCP_ENABLED: ${MCP_ENABLED:-false}

docker-compose.prod.yml

@@ -22,6 +22,8 @@ services:
     mem_limit: 3g
     environment:
       REDIS_URL: redis://redis:6379
+      REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-5000}
+      REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-5000}
       BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-3}
       BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0}
       MCP_ENABLED: ${MCP_ENABLED:-false}

docker-compose.yml

@@ -22,6 +22,8 @@ services:
     shm_size: 1gb
     environment:
       REDIS_URL: redis://redis:6379
+      REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-5000}
+      REDIS_RETRY_DELAY_MS: ${REDIS_RETRY_DELAY_MS:-5000}
       BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-1}
       BROWSER_HEADFUL_POOL_SIZE: ${BROWSER_HEADFUL_POOL_SIZE:-0}
       MCP_ENABLED: ${MCP_ENABLED:-false}