germondai · emandel2630 · Aug 19, 2026
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Optional blocked-outcome evidence: `blockedEvidence: true` on `POST /scrape` attaches the challenge wall the deepest browser tier stopped at — `html`, `url`, `statusCode`, `reason`, and the `screenshot` when one was requested — to the **500 body**, alongside the existing `error` and `timings`. A blocked scrape stays a failure and `ScrapeResult` never carries a wall as content (#53); this only makes the failure diagnosable and keeps the page as evidence. Off by default — without the flag nothing is read on the branch that gives up. Failures with no page to read (Tier 1, a context or page that never opened, a hard network failure, `about:neterror`, an empty document, pool exhaustion) carry `timings` only. The wall's markup is capped by `BLOCKED_EVIDENCE_MAX_HTML_CHARS` and flagged `htmlTruncated` past it, and a capture failure leaves the evidence off the body rather than changing the outcome.
+
 ## [1.5.0] - 2026-09-04
 
 ### Changed

@@ -0,0 +1,102 @@
+import { describe, expect, test } from "bun:test"
+import type { BrowserHandle } from "@trawl/browser"
+import type { OrchestratorDeps } from "@trawl/tiers"
+import type { SessionData } from "@trawl/types"
+import { scrapeRoute } from "./scrape"
+
+const WALL_HTML = `<html><head><title>Access denied</title></head><body><h1>403</h1>${"blocked ".repeat(40)}</body></html>`
+const JPEG_BASE64 = Buffer.from("fake-jpeg-bytes").toString("base64")
+
+const session: SessionData = { cookies: [], userAgent: "cached-user-agent", savedAt: 1 }
+
+const mainFrame = {}
+const wallPage = {
+  url: () => "https://example.com/blocked",
+  title: async () => "Access denied",
+  content: async () => WALL_HTML,
+  goto: async () => {},
+  on: (event: string, handler: (response: unknown) => void) => {
+    if (event !== "response") return
+    handler({
+      url: () => "https://example.com/blocked",
+      status: () => 403,
+      headers: () => ({}),
+      body: async () => Buffer.from(WALL_HTML),
+      request: () => ({ isNavigationRequest: () => true, frame: () => mainFrame }),
+    })
+  },
+  off: () => {},
+  once: () => {},
+  mainFrame: () => mainFrame,
+  frames: () => [],
+  context: () => ({ cookies: async () => [] }),
+  evaluate: async () => "test-agent",
+  setExtraHTTPHeaders: async () => {},
+  waitForLoadState: async () => {},
+  close: async () => {},
+  screenshot: async () => Buffer.from("fake-jpeg-bytes"),
+}
+
+const blockedDeps = (): OrchestratorDeps => ({
+  acquireBrowser: async () =>
+    ({
+      id: 1,
+      lease: 1,
+      headful: false,
+      context: { newPage: async () => wallPage, addCookies: async () => {}, cookies: async () => [] },
+      browser: {},
+      fingerprint: { userAgent: "test-agent", platform: "Linux x86_64", locale: "en-US", timezone: "UTC" },
+    }) satisfies BrowserHandle,
+  releaseBrowser: () => {},
+  loadSession: async () => session,
+  saveSession: async () => {},
+  invalidateSession: async () => {},
+})
+
+const post = (body: unknown) =>
+  scrapeRoute(blockedDeps, () => ({})).handle(
+    new Request("http://localhost/scrape", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+    }),
+  )
+
+const blockedRequest = { url: "https://example.com", skipHttp: true, maxTier: 2, maxTimeout: 4_000 }
+
+describe("POST /scrape on a blocked outcome", () => {
+  test("still answers 500 with the per-tier attempt history", async () => {
+    const response = await post(blockedRequest)
+
+    expect(response.status).toBe(500)
+    const body = await response.json()
+    expect(body.error).toContain("Max tier reached without success")
+    expect(body.timings).toEqual([{ tier: 2, status: "blocked", durationMs: expect.any(Number), reason: "http-403" }])
+    expect(body.blockedEvidence).toBeUndefined()
+  })
+
+  test("carries the challenge wall when the request asked for it", async () => {
+    const response = await post({ ...blockedRequest, blockedEvidence: true, screenshot: true })
+
+    expect(response.status).toBe(500)
+    const body = await response.json()
+    expect(body.timings[0].reason).toBe("http-403")
+    expect(body.blockedEvidence).toMatchObject({
+      tier: 2,
+      status: "blocked",
+      reason: "http-403",
+      url: "https://example.com/blocked",
+      statusCode: 403,
+      screenshot: JPEG_BASE64,
+    })
+    expect(body.blockedEvidence.html).toContain("Access denied")
+  })
+
+  test("omits the image when only the markup was asked for", async () => {
+    const response = await post({ ...blockedRequest, blockedEvidence: true })
+
+    const body = await response.json()
+    expect(body.blockedEvidence.html).toContain("Access denied")
+    expect(body.blockedEvidence.screenshot).toBeUndefined()
+  })
+})
@@ -1,4 +1,5 @@
 import { PoolExhaustedError } from "@trawl/browser"
+import type { OrchestratorDeps } from "@trawl/tiers"
 import { RequestValidationError, ScrapeError, sanitizeHeaders, scrape } from "@trawl/tiers"
 import type { ScrapeRequest } from "@trawl/types"
 import { Elysia } from "elysia"
@@ -10,17 +11,17 @@ import { requestUrl, validateScrapeRequest } from "../validation"
 // Error mapping:
 //   503 — pool still initializing (native { error })
 //   429 — pool exhausted (FlareSolverr envelope; uniform with /v1)
-//   500 — other scrape exception (native { error })
-export function scrapeRoute() {
+//   500 — other scrape exception (native { error, timings, blockedEvidence })
+export function scrapeRoute(deps: () => OrchestratorDeps = getDeps, poolReady: () => unknown = getPool) {
   return new Elysia().post("/scrape", async ({ body, set }) => {
     try {
       validateScrapeRequest(body)
       const req: ScrapeRequest = body
-      if (!getPool()) {
+      if (!poolReady()) {
         set.status = 503
         return { error: "Browser pool initializing, retry in a few seconds" }
       }
-      return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, getDeps())
+      return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, deps())
     } catch (err) {
       if (err instanceof RequestValidationError) {
         set.status = err.statusCode
@@ -32,7 +33,7 @@ export function scrapeRoute() {
       }
       set.status = 500
       if (err instanceof ScrapeError) {
-        return { error: err.message, timings: err.timings }
+        return { error: err.message, timings: err.timings, blockedEvidence: err.blockedEvidence }
       }
       return { error: err instanceof Error ? err.message : String(err) }
     }

@@ -25,6 +25,7 @@ interface ScrapeRequest {
   captureResponses?: string[]            // URL patterns whose response bodies to capture, default none
   settleTimeout?: number                 // ms to wait after load for a match, default 15000
   waitForSelector?: string               // CSS selector that ends the settle window early
+  blockedEvidence?: boolean              // return the challenge wall on the error, default false
 }
 ```
 
@@ -46,6 +47,7 @@ interface ScrapeRequest {
 | `captureResponses` | string[] | — | URL patterns — a substring, or a glob matched against the whole URL when the pattern contains `*` or `?` — whose response bodies are returned as `capturedResponses` (browser tiers 2–4)     |
 | `settleTimeout` | number | 15000  | Milliseconds to hold the page open after load waiting for a match; ends early on the first captured body, on `waitForSelector`, or on network idle. Only read alongside `captureResponses`  |
 | `waitForSelector` | string | —    | CSS selector that also ends the settle window early. Only read alongside `captureResponses`                                                                                                 |
+| `blockedEvidence` | boolean | false | When no tier clears the challenge, attach the wall the last browser tier stopped at to the 500 body as `blockedEvidence`. It is never attached to a successful result — see the note below. The image rides along only when `screenshot` is also set |
 
 Captured response bodies, headers, console messages, and URLs can contain credentials,
 tokens, or personal data. Treat these opt-in diagnostic fields as sensitive output.
@@ -108,6 +110,37 @@ interface TierResult {
 }
 ```
 
+## Blocked-Outcome Evidence
+
+A scrape that runs a browser but never clears the challenge is still a failure: it answers
+**500**, and `ScrapeResult` never carries a challenge wall dressed up as content.
+
+`blockedEvidence: true` attaches the wall to that failure instead, so a caller can tell
+"blocked by a challenge" from "TRAWL broke" and can keep the page as evidence:
+
+```typescript
+interface BlockedEvidence {
+  tier: 2 | 3 | 4              // which browser tier rendered the wall
+  status: 'blocked' | 'timeout'
+  reason?: string              // identical to the matching timings[].reason
+  url: string                  // where the browser stood, after any challenge redirects
+  statusCode?: number
+  html?: string                // the wall's markup
+  htmlTruncated?: boolean      // html is the head of a page over BLOCKED_EVIDENCE_MAX_HTML_CHARS
+  screenshot?: string          // base64 JPEG, only when `screenshot` was also requested
+}
+```
+
+The wall reported is the one from the **deepest** browser tier that rendered one — a Tier 3
+wall is replaced by Tier 4's when Tier 4 also fails. Some failures have no page to hand
+back at all and carry no evidence: Tier 1 (a plain HTTP fetch, no browser), a tier that
+could not open a context or a page, a hard network failure (DNS, connection refused, TLS),
+an `about:neterror` page, an empty document, and a pool that was exhausted or still
+initializing. In those cases `timings` alone tells the story.
+
+Reading the wall never changes the outcome: a capture failure leaves `blockedEvidence` off
+the body and logs the reason, and the status code and `timings` are the same either way.
+
 ## Examples
 
 ### Minimal request
@@ -219,3 +252,27 @@ alone — no need to check server logs:
   ]
 }
 ```
+
+When the request set `blockedEvidence: true` and a browser tier rendered a challenge wall,
+the same body also carries that page — see
+[Blocked-Outcome Evidence](#blocked-outcome-evidence):
+
+```json
+{
+  "error": "All tiers exhausted. Last failure: cloudflare-persistent",
+  "timings": [
+    { "tier": 1, "status": "needs-js", "durationMs": 50, "reason": "cloudflare-challenge" },
+    { "tier": 3, "status": "blocked", "durationMs": 2942, "reason": "cloudflare-persistent" },
+    { "tier": 4, "status": "blocked", "durationMs": 7890, "reason": "cloudflare-persistent" }
+  ],
+  "blockedEvidence": {
+    "tier": 4,
+    "status": "blocked",
+    "reason": "cloudflare-persistent",
+    "url": "https://nowsecure.nl/",
+    "statusCode": 403,
+    "html": "<!DOCTYPE html><html><head><title>Just a moment...</title>...",
+    "screenshot": "/9j/4AAQSkZJRgABAQAA..."
+  }
+}
+```
@@ -258,6 +258,24 @@ bodies with a valid `Content-Length` are read. Compressed or unknown-size bodies
 returned with `body: null` and an error. Declared sizes are reserved cumulatively before
 reads start, so concurrent responses cannot exceed the total read budget.
 
+## Blocked-Outcome Evidence
+
+Only read when a request sets `blockedEvidence: true` — see
+[Native API](/api-reference/native-api#blocked-outcome-evidence). Without it no page is
+read on the branch that gives up. One wall is kept per request, so its markup is the only
+unbounded dimension.
+
+| Variable | Default | Purpose |
+| --- | ---: | --- |
+| `BLOCKED_EVIDENCE_MAX_HTML_CHARS` | `512000` | Characters of the wall kept; past it `html` is the head of the page and `htmlTruncated` is set |
+
+The wall is truncated rather than dropped: unlike a stylesheet or an image, the head of a
+challenge page still carries the title, the vendor markers and the incident id a caller
+classifies on. Reading it never fails the scrape — a capture failure leaves the evidence
+off the error and logs the reason, and the HTTP status and `timings` are unchanged either
+way.
+
+
 ## CAPTCHA audio and media tools
 
 TRAWL uses ffmpeg while solving supported CAPTCHA challenges. reCAPTCHA audio is converted before

@@ -1,6 +1,6 @@
 import type { BrowserHandle } from "@trawl/browser"
 import { FINGERPRINT, FINGERPRINT_POOL } from "@trawl/browser"
-import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types"
+import type { BlockedEvidence, Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types"
 import { runTier1 } from "./tiers/1"
 import { runTier2 } from "./tiers/2"
 import { runTier3 } from "./tiers/3"
@@ -20,10 +20,15 @@ const MAX_PROXY_ATTEMPTS = 2
 // wasn't reaching anyone outside the orchestrator.
 export class ScrapeError extends Error {
   timings: TierResult[]
-  constructor(message: string, timings: TierResult[]) {
+  // The challenge wall the last browser tier stopped at, when the caller asked for it.
+  // It rides the error rather than a `blocked` ScrapeResult on purpose: a wall is not a
+  // scrape, and callers keyed on the success shape must never be handed one.
+  blockedEvidence?: BlockedEvidence
+  constructor(message: string, timings: TierResult[], blockedEvidence?: BlockedEvidence) {
     super(message)
     this.name = "ScrapeError"
     this.timings = timings
+    this.blockedEvidence = blockedEvidence
   }
 }
 
@@ -76,13 +81,25 @@ export async function scrape(
   const tier1Proxy = explicitProxy && /^https?:\/\//i.test(explicitProxy) ? explicitProxy : undefined
   const skipTier1ForProxy = Boolean(explicitProxy && !tier1Proxy)
 
+  // Evidence from the last browser tier that rendered a wall it could not clear. Kept out
+  // of `timings` — that stays the thin, machine-readable attempt history — and reached
+  // only via the thrown ScrapeError.
+  let blockedEvidence: BlockedEvidence | undefined
   const capture = {
     consoleLogs: req.consoleLogs,
     networkLogs: req.networkLogs,
     redirectChain: req.redirectChain,
     captureResponses: req.captureResponses,
     settleTimeout: req.settleTimeout,
     waitForSelector: req.waitForSelector,
+    blockedEvidence: req.blockedEvidence
+      ? {
+          screenshot: req.screenshot,
+          report: (evidence: BlockedEvidence) => {
+            blockedEvidence = evidence
+          },
+        }
+      : undefined,
   }
 
   const sanitizedHeaders = sanitizeHeaders(req.headers)
@@ -146,7 +163,7 @@ export async function scrape(
   }
 
   if (maxTier < 2) {
-    throw new ScrapeError("Max tier reached without success", timings)
+    throw new ScrapeError("Max tier reached without success", timings, blockedEvidence)
   }
 
   // Acquire browser for tiers 2-4
@@ -232,7 +249,7 @@ export async function scrape(
     }
 
     if (maxTier < 3) {
-      throw new ScrapeError("Max tier reached without success", timings)
+      throw new ScrapeError("Max tier reached without success", timings, blockedEvidence)
     }
 
     // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request
@@ -317,7 +334,7 @@ export async function scrape(
     }
 
     if (maxTier < 4) {
-      throw new ScrapeError("Max tier reached without success", timings)
+      throw new ScrapeError("Max tier reached without success", timings, blockedEvidence)
     }
 
     // Tier 4: residential proxy escalation — requires at least one residential proxy,
@@ -327,6 +344,7 @@ export async function scrape(
       throw new ScrapeError(
         `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL (or pass a proxy per-request) to enable Tier 4 proxy escalation.`,
         timings,
+        blockedEvidence,
       )
     }
 
@@ -403,7 +421,7 @@ export async function scrape(
       }
     }
 
-    throw new ScrapeError(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`, timings)
+    throw new ScrapeError(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`, timings, blockedEvidence)
   } finally {
     if (!handleReleased) deps.releaseBrowser(handle)
   }

@@ -16,18 +16,22 @@ const MAX_BYTES = captureLimit(process.env.SCREENSHOT_MAX_BYTES, 4_000_000)
 export async function capturePageScreenshot(
   page: Page,
   budgetMs = Number.POSITIVE_INFINITY,
+  options: { settle?: boolean } = {},
 ): Promise<string | undefined> {
   const deadline = Date.now() + Math.max(budgetMs, 0)
   const remaining = (): number => Math.max(deadline - Date.now(), 0)
 
   try {
     // The HTML is read the moment a challenge clears, before late content (images,
     // fonts, lazy hydration) has painted. Give the page a bounded chance to settle,
-    // then a short beat for whatever paints after the last request.
+    // then a short beat for whatever paints after the last request. A caller imaging a
+    // page that will never settle (a challenge wall) opts out of the wait.
     if (remaining() <= 0) return undefined
-    await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, remaining()) }).catch(() => {})
-    const paintWaitMs = Math.min(300, remaining())
-    if (paintWaitMs > 0) await new Promise((r) => setTimeout(r, paintWaitMs))
+    if (options.settle !== false) {
+      await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, remaining()) }).catch(() => {})
+      const paintWaitMs = Math.min(300, remaining())
+      if (paintWaitMs > 0) await new Promise((r) => setTimeout(r, paintWaitMs))
+    }
 
     const captureTimeout = Math.min(CAPTURE_TIMEOUT_MS, remaining())
     if (captureTimeout <= 0) return undefined