From 96a318197e34c457c6f2faeb5bdcb882dcca0727 Mon Sep 17 00:00:00 2001 From: emandel2630 Date: Wed, 19 Aug 2026 12:42:25 -0400 Subject: [PATCH] feat(scrape): optional challenge-wall evidence on the blocked error path A scrape that ran a browser but never cleared the challenge answers HTTP 500 with {error, timings} and discards the wall it stopped at. `timings[].reason` says a challenge persisted, but there is no way to see it, and "blocked by a wall" is indistinguishable from "TRAWL broke" without reading server logs. `blockedEvidence: true` attaches the wall to that failure: `blockedEvidence` on ScrapeError, serialized into the 500 body next to the existing `error` and `timings` - tier, status, reason (the same string as the matching timing), the landing url, statusCode, the markup, and the screenshot when one was requested. The wall from the deepest browser tier that rendered one wins. Deliberately not the blocked-as-success shape #53 moved away from. A blocked scrape stays a failure: the status stays 500, and ScrapeResult never carries a challenge wall as content. Only the error becomes diagnosable. - Opt-in and terminal-path only: without the flag the sink is never built, so reportBlocked returns before reading a page or taking an image, and a stock request is unchanged. The image needs `screenshot` as well. - Bounded: one wall per request, its markup capped at BLOCKED_EVIDENCE_MAX_HTML_CHARS (512000) and flagged `htmlTruncated`. The head of a wall carries the title, vendor markers and incident id a caller classifies on, so this one truncates rather than dropping. - Contained: reportBlocked catches everything and logs - a failed read leaves the evidence off the error and never changes a tier's outcome. It holds no page reference past its own call; only strings reach the orchestrator. - Branches with no page to read carry timings only: Tier 1, a context or page that never opened, a hard network failure, about:neterror, an empty document, a saturated pool. - capturePageScreenshot gains `settle: false` for these - a challenge wall never reaches network idle, so the settle wait only spends the next tier's budget. The remaining-budget bound from #106 is kept. --- CHANGELOG.md | 3 + apps/api/src/routes/scrape.test.ts | 102 +++++++ apps/api/src/routes/scrape.ts | 11 +- apps/docs/api-reference/native-api.md | 57 ++++ apps/docs/getting-started/configuration.md | 18 ++ packages/tiers/src/orchestrator.ts | 30 +- packages/tiers/src/screenshot.ts | 12 +- packages/tiers/src/tiers/2.ts | 33 ++- packages/tiers/src/tiers/3.ts | 56 +++- packages/tiers/src/tiers/4.ts | 56 +++- packages/tiers/src/utils/blockedEvidence.ts | 53 ++++ packages/tiers/src/utils/capture.ts | 5 +- packages/tiers/tests/blockedEvidence.test.ts | 287 +++++++++++++++++++ packages/types/src/index.ts | 23 ++ 14 files changed, 705 insertions(+), 41 deletions(-) create mode 100644 apps/api/src/routes/scrape.test.ts create mode 100644 packages/tiers/src/utils/blockedEvidence.ts create mode 100644 packages/tiers/tests/blockedEvidence.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 64456af..cbe9759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Optional blocked-outcome evidence: `blockedEvidence: true` on `POST /scrape` attaches the challenge wall the deepest browser tier stopped at — `html`, `url`, `statusCode`, `reason`, and the `screenshot` when one was requested — to the **500 body**, alongside the existing `error` and `timings`. A blocked scrape stays a failure and `ScrapeResult` never carries a wall as content (#53); this only makes the failure diagnosable and keeps the page as evidence. Off by default — without the flag nothing is read on the branch that gives up. Failures with no page to read (Tier 1, a context or page that never opened, a hard network failure, `about:neterror`, an empty document, pool exhaustion) carry `timings` only. The wall's markup is capped by `BLOCKED_EVIDENCE_MAX_HTML_CHARS` and flagged `htmlTruncated` past it, and a capture failure leaves the evidence off the body rather than changing the outcome. + ## [1.5.0] - 2026-09-04 ### Changed diff --git a/apps/api/src/routes/scrape.test.ts b/apps/api/src/routes/scrape.test.ts new file mode 100644 index 0000000..2c09bd8 --- /dev/null +++ b/apps/api/src/routes/scrape.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, test } from "bun:test" +import type { BrowserHandle } from "@trawl/browser" +import type { OrchestratorDeps } from "@trawl/tiers" +import type { SessionData } from "@trawl/types" +import { scrapeRoute } from "./scrape" + +const WALL_HTML = `Access denied

403

${"blocked ".repeat(40)}` +const JPEG_BASE64 = Buffer.from("fake-jpeg-bytes").toString("base64") + +const session: SessionData = { cookies: [], userAgent: "cached-user-agent", savedAt: 1 } + +const mainFrame = {} +const wallPage = { + url: () => "https://example.com/blocked", + title: async () => "Access denied", + content: async () => WALL_HTML, + goto: async () => {}, + on: (event: string, handler: (response: unknown) => void) => { + if (event !== "response") return + handler({ + url: () => "https://example.com/blocked", + status: () => 403, + headers: () => ({}), + body: async () => Buffer.from(WALL_HTML), + request: () => ({ isNavigationRequest: () => true, frame: () => mainFrame }), + }) + }, + off: () => {}, + once: () => {}, + mainFrame: () => mainFrame, + frames: () => [], + context: () => ({ cookies: async () => [] }), + evaluate: async () => "test-agent", + setExtraHTTPHeaders: async () => {}, + waitForLoadState: async () => {}, + close: async () => {}, + screenshot: async () => Buffer.from("fake-jpeg-bytes"), +} + +const blockedDeps = (): OrchestratorDeps => ({ + acquireBrowser: async () => + ({ + id: 1, + lease: 1, + headful: false, + context: { newPage: async () => wallPage, addCookies: async () => {}, cookies: async () => [] }, + browser: {}, + fingerprint: { userAgent: "test-agent", platform: "Linux x86_64", locale: "en-US", timezone: "UTC" }, + }) satisfies BrowserHandle, + releaseBrowser: () => {}, + loadSession: async () => session, + saveSession: async () => {}, + invalidateSession: async () => {}, +}) + +const post = (body: unknown) => + scrapeRoute(blockedDeps, () => ({})).handle( + new Request("http://localhost/scrape", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ) + +const blockedRequest = { url: "https://example.com", skipHttp: true, maxTier: 2, maxTimeout: 4_000 } + +describe("POST /scrape on a blocked outcome", () => { + test("still answers 500 with the per-tier attempt history", async () => { + const response = await post(blockedRequest) + + expect(response.status).toBe(500) + const body = await response.json() + expect(body.error).toContain("Max tier reached without success") + expect(body.timings).toEqual([{ tier: 2, status: "blocked", durationMs: expect.any(Number), reason: "http-403" }]) + expect(body.blockedEvidence).toBeUndefined() + }) + + test("carries the challenge wall when the request asked for it", async () => { + const response = await post({ ...blockedRequest, blockedEvidence: true, screenshot: true }) + + expect(response.status).toBe(500) + const body = await response.json() + expect(body.timings[0].reason).toBe("http-403") + expect(body.blockedEvidence).toMatchObject({ + tier: 2, + status: "blocked", + reason: "http-403", + url: "https://example.com/blocked", + statusCode: 403, + screenshot: JPEG_BASE64, + }) + expect(body.blockedEvidence.html).toContain("Access denied") + }) + + test("omits the image when only the markup was asked for", async () => { + const response = await post({ ...blockedRequest, blockedEvidence: true }) + + const body = await response.json() + expect(body.blockedEvidence.html).toContain("Access denied") + expect(body.blockedEvidence.screenshot).toBeUndefined() + }) +}) diff --git a/apps/api/src/routes/scrape.ts b/apps/api/src/routes/scrape.ts index 9a359f5..97b6384 100644 --- a/apps/api/src/routes/scrape.ts +++ b/apps/api/src/routes/scrape.ts @@ -1,4 +1,5 @@ import { PoolExhaustedError } from "@trawl/browser" +import type { OrchestratorDeps } from "@trawl/tiers" import { RequestValidationError, ScrapeError, sanitizeHeaders, scrape } from "@trawl/tiers" import type { ScrapeRequest } from "@trawl/types" import { Elysia } from "elysia" @@ -10,17 +11,17 @@ import { requestUrl, validateScrapeRequest } from "../validation" // Error mapping: // 503 — pool still initializing (native { error }) // 429 — pool exhausted (FlareSolverr envelope; uniform with /v1) -// 500 — other scrape exception (native { error }) -export function scrapeRoute() { +// 500 — other scrape exception (native { error, timings, blockedEvidence }) +export function scrapeRoute(deps: () => OrchestratorDeps = getDeps, poolReady: () => unknown = getPool) { return new Elysia().post("/scrape", async ({ body, set }) => { try { validateScrapeRequest(body) const req: ScrapeRequest = body - if (!getPool()) { + if (!poolReady()) { set.status = 503 return { error: "Browser pool initializing, retry in a few seconds" } } - return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, getDeps()) + return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, deps()) } catch (err) { if (err instanceof RequestValidationError) { set.status = err.statusCode @@ -32,7 +33,7 @@ export function scrapeRoute() { } set.status = 500 if (err instanceof ScrapeError) { - return { error: err.message, timings: err.timings } + return { error: err.message, timings: err.timings, blockedEvidence: err.blockedEvidence } } return { error: err instanceof Error ? err.message : String(err) } } diff --git a/apps/docs/api-reference/native-api.md b/apps/docs/api-reference/native-api.md index b53701f..7e9e4fe 100644 --- a/apps/docs/api-reference/native-api.md +++ b/apps/docs/api-reference/native-api.md @@ -25,6 +25,7 @@ interface ScrapeRequest { captureResponses?: string[] // URL patterns whose response bodies to capture, default none settleTimeout?: number // ms to wait after load for a match, default 15000 waitForSelector?: string // CSS selector that ends the settle window early + blockedEvidence?: boolean // return the challenge wall on the error, default false } ``` @@ -46,6 +47,7 @@ interface ScrapeRequest { | `captureResponses` | string[] | — | URL patterns — a substring, or a glob matched against the whole URL when the pattern contains `*` or `?` — whose response bodies are returned as `capturedResponses` (browser tiers 2–4) | | `settleTimeout` | number | 15000 | Milliseconds to hold the page open after load waiting for a match; ends early on the first captured body, on `waitForSelector`, or on network idle. Only read alongside `captureResponses` | | `waitForSelector` | string | — | CSS selector that also ends the settle window early. Only read alongside `captureResponses` | +| `blockedEvidence` | boolean | false | When no tier clears the challenge, attach the wall the last browser tier stopped at to the 500 body as `blockedEvidence`. It is never attached to a successful result — see the note below. The image rides along only when `screenshot` is also set | Captured response bodies, headers, console messages, and URLs can contain credentials, tokens, or personal data. Treat these opt-in diagnostic fields as sensitive output. @@ -108,6 +110,37 @@ interface TierResult { } ``` +## Blocked-Outcome Evidence + +A scrape that runs a browser but never clears the challenge is still a failure: it answers +**500**, and `ScrapeResult` never carries a challenge wall dressed up as content. + +`blockedEvidence: true` attaches the wall to that failure instead, so a caller can tell +"blocked by a challenge" from "TRAWL broke" and can keep the page as evidence: + +```typescript +interface BlockedEvidence { + tier: 2 | 3 | 4 // which browser tier rendered the wall + status: 'blocked' | 'timeout' + reason?: string // identical to the matching timings[].reason + url: string // where the browser stood, after any challenge redirects + statusCode?: number + html?: string // the wall's markup + htmlTruncated?: boolean // html is the head of a page over BLOCKED_EVIDENCE_MAX_HTML_CHARS + screenshot?: string // base64 JPEG, only when `screenshot` was also requested +} +``` + +The wall reported is the one from the **deepest** browser tier that rendered one — a Tier 3 +wall is replaced by Tier 4's when Tier 4 also fails. Some failures have no page to hand +back at all and carry no evidence: Tier 1 (a plain HTTP fetch, no browser), a tier that +could not open a context or a page, a hard network failure (DNS, connection refused, TLS), +an `about:neterror` page, an empty document, and a pool that was exhausted or still +initializing. In those cases `timings` alone tells the story. + +Reading the wall never changes the outcome: a capture failure leaves `blockedEvidence` off +the body and logs the reason, and the status code and `timings` are the same either way. + ## Examples ### Minimal request @@ -219,3 +252,27 @@ alone — no need to check server logs: ] } ``` + +When the request set `blockedEvidence: true` and a browser tier rendered a challenge wall, +the same body also carries that page — see +[Blocked-Outcome Evidence](#blocked-outcome-evidence): + +```json +{ + "error": "All tiers exhausted. Last failure: cloudflare-persistent", + "timings": [ + { "tier": 1, "status": "needs-js", "durationMs": 50, "reason": "cloudflare-challenge" }, + { "tier": 3, "status": "blocked", "durationMs": 2942, "reason": "cloudflare-persistent" }, + { "tier": 4, "status": "blocked", "durationMs": 7890, "reason": "cloudflare-persistent" } + ], + "blockedEvidence": { + "tier": 4, + "status": "blocked", + "reason": "cloudflare-persistent", + "url": "https://nowsecure.nl/", + "statusCode": 403, + "html": "Just a moment......", + "screenshot": "/9j/4AAQSkZJRgABAQAA..." + } +} +``` diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index e95cbdc..e4ed28e 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -258,6 +258,24 @@ bodies with a valid `Content-Length` are read. Compressed or unknown-size bodies returned with `body: null` and an error. Declared sizes are reserved cumulatively before reads start, so concurrent responses cannot exceed the total read budget. +## Blocked-Outcome Evidence + +Only read when a request sets `blockedEvidence: true` — see +[Native API](/api-reference/native-api#blocked-outcome-evidence). Without it no page is +read on the branch that gives up. One wall is kept per request, so its markup is the only +unbounded dimension. + +| Variable | Default | Purpose | +| --- | ---: | --- | +| `BLOCKED_EVIDENCE_MAX_HTML_CHARS` | `512000` | Characters of the wall kept; past it `html` is the head of the page and `htmlTruncated` is set | + +The wall is truncated rather than dropped: unlike a stylesheet or an image, the head of a +challenge page still carries the title, the vendor markers and the incident id a caller +classifies on. Reading it never fails the scrape — a capture failure leaves the evidence +off the error and logs the reason, and the HTTP status and `timings` are unchanged either +way. + + ## CAPTCHA audio and media tools TRAWL uses ffmpeg while solving supported CAPTCHA challenges. reCAPTCHA audio is converted before diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 81fd9c6..b687bba 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -1,6 +1,6 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, FINGERPRINT_POOL } from "@trawl/browser" -import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" +import type { BlockedEvidence, Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" import { runTier1 } from "./tiers/1" import { runTier2 } from "./tiers/2" import { runTier3 } from "./tiers/3" @@ -20,10 +20,15 @@ const MAX_PROXY_ATTEMPTS = 2 // wasn't reaching anyone outside the orchestrator. export class ScrapeError extends Error { timings: TierResult[] - constructor(message: string, timings: TierResult[]) { + // The challenge wall the last browser tier stopped at, when the caller asked for it. + // It rides the error rather than a `blocked` ScrapeResult on purpose: a wall is not a + // scrape, and callers keyed on the success shape must never be handed one. + blockedEvidence?: BlockedEvidence + constructor(message: string, timings: TierResult[], blockedEvidence?: BlockedEvidence) { super(message) this.name = "ScrapeError" this.timings = timings + this.blockedEvidence = blockedEvidence } } @@ -76,6 +81,10 @@ export async function scrape( const tier1Proxy = explicitProxy && /^https?:\/\//i.test(explicitProxy) ? explicitProxy : undefined const skipTier1ForProxy = Boolean(explicitProxy && !tier1Proxy) + // Evidence from the last browser tier that rendered a wall it could not clear. Kept out + // of `timings` — that stays the thin, machine-readable attempt history — and reached + // only via the thrown ScrapeError. + let blockedEvidence: BlockedEvidence | undefined const capture = { consoleLogs: req.consoleLogs, networkLogs: req.networkLogs, @@ -83,6 +92,14 @@ export async function scrape( captureResponses: req.captureResponses, settleTimeout: req.settleTimeout, waitForSelector: req.waitForSelector, + blockedEvidence: req.blockedEvidence + ? { + screenshot: req.screenshot, + report: (evidence: BlockedEvidence) => { + blockedEvidence = evidence + }, + } + : undefined, } const sanitizedHeaders = sanitizeHeaders(req.headers) @@ -146,7 +163,7 @@ export async function scrape( } if (maxTier < 2) { - throw new ScrapeError("Max tier reached without success", timings) + throw new ScrapeError("Max tier reached without success", timings, blockedEvidence) } // Acquire browser for tiers 2-4 @@ -232,7 +249,7 @@ export async function scrape( } if (maxTier < 3) { - throw new ScrapeError("Max tier reached without success", timings) + throw new ScrapeError("Max tier reached without success", timings, blockedEvidence) } // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request @@ -317,7 +334,7 @@ export async function scrape( } if (maxTier < 4) { - throw new ScrapeError("Max tier reached without success", timings) + throw new ScrapeError("Max tier reached without success", timings, blockedEvidence) } // Tier 4: residential proxy escalation — requires at least one residential proxy, @@ -327,6 +344,7 @@ export async function scrape( throw new ScrapeError( `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL (or pass a proxy per-request) to enable Tier 4 proxy escalation.`, timings, + blockedEvidence, ) } @@ -403,7 +421,7 @@ export async function scrape( } } - throw new ScrapeError(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`, timings) + throw new ScrapeError(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`, timings, blockedEvidence) } finally { if (!handleReleased) deps.releaseBrowser(handle) } diff --git a/packages/tiers/src/screenshot.ts b/packages/tiers/src/screenshot.ts index 3f3a058..240a15a 100644 --- a/packages/tiers/src/screenshot.ts +++ b/packages/tiers/src/screenshot.ts @@ -16,6 +16,7 @@ const MAX_BYTES = captureLimit(process.env.SCREENSHOT_MAX_BYTES, 4_000_000) export async function capturePageScreenshot( page: Page, budgetMs = Number.POSITIVE_INFINITY, + options: { settle?: boolean } = {}, ): Promise { const deadline = Date.now() + Math.max(budgetMs, 0) const remaining = (): number => Math.max(deadline - Date.now(), 0) @@ -23,11 +24,14 @@ export async function capturePageScreenshot( try { // The HTML is read the moment a challenge clears, before late content (images, // fonts, lazy hydration) has painted. Give the page a bounded chance to settle, - // then a short beat for whatever paints after the last request. + // then a short beat for whatever paints after the last request. A caller imaging a + // page that will never settle (a challenge wall) opts out of the wait. if (remaining() <= 0) return undefined - await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, remaining()) }).catch(() => {}) - const paintWaitMs = Math.min(300, remaining()) - if (paintWaitMs > 0) await new Promise((r) => setTimeout(r, paintWaitMs)) + if (options.settle !== false) { + await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, remaining()) }).catch(() => {}) + const paintWaitMs = Math.min(300, remaining()) + if (paintWaitMs > 0) await new Promise((r) => setTimeout(r, paintWaitMs)) + } const captureTimeout = Math.min(CAPTURE_TIMEOUT_MS, remaining()) if (captureTimeout <= 0) return undefined diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index 237506b..3a45af8 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -9,6 +9,7 @@ import type { } from "@trawl/types" import { capturePageScreenshot } from "../screenshot" import { solvePageCaptchas } from "../solvers" +import { reportBlocked } from "../utils/blockedEvidence" import { attachPageCapture, type CaptureOptions } from "../utils/capture" import { normalizeSameSite, toCookies } from "../utils/cookies" import { @@ -105,12 +106,26 @@ export async function runTier2( } if (isCloudflarePage(html, mainResponse.headers)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason: "session-expired", + statusCode: mainResponse.status, + html, + }) return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } // A cached session that lands back on Akamai's interstitial is stale — force a // fresh Tier-3 solve rather than returning the ~2KB challenge stub as content. if (hasAkamaiChallenge(html)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason: "akamai-session-expired", + statusCode: mainResponse.status, + html, + }) return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "akamai-session-expired" } } @@ -127,7 +142,15 @@ export async function runTier2( } if (isBlocked(mainResponse.status, html)) { - return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } + const reason = `http-${mainResponse.status}` + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason, + statusCode: mainResponse.status, + html, + }) + return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason } } // Attempt to solve any embedded captcha widgets (Turnstile, reCAPTCHA, hCaptcha). @@ -150,6 +173,14 @@ export async function runTier2( const finalHtml = await page.content() if (isCloudflarePage(finalHtml, mainResponse.headers)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason: "session-expired", + statusCode: mainResponse.status, + html: finalHtml, + screenshot: shot, + }) return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } diff --git a/packages/tiers/src/tiers/3.ts b/packages/tiers/src/tiers/3.ts index 2800527..c731792 100644 --- a/packages/tiers/src/tiers/3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -3,6 +3,7 @@ import { closeTemporaryContext, FINGERPRINT, newFreshContext } from "@trawl/brow import type { CapturedResponseEntry, ConsoleLogEntry, Cookie, NetworkLogEntry, TierResult } from "@trawl/types" import { capturePageScreenshot } from "../screenshot" import { solvePageCaptchas } from "../solvers" +import { reportBlocked } from "../utils/blockedEvidence" import { attachPageCapture, type CaptureOptions } from "../utils/capture" import { routeChallengeWait } from "../utils/challengeRouter" import { snapshotChallengeCookies, toCookies } from "../utils/cookies" @@ -138,17 +139,15 @@ export async function runTier3( ) if (resolution !== "ok") { - return { - tier: 3, - status: resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout", - durationMs: Date.now() - start, - reason: - resolution === "captcha-required" - ? `${challengeType}-captcha-required` - : resolution === "ip-blocked" - ? (DATACENTER_BLOCKED_REASONS[challengeType] ?? DEFAULT_DATACENTER_BLOCKED_REASON) - : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, - } + const status = resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout" + const reason = + resolution === "captcha-required" + ? `${challengeType}-captcha-required` + : resolution === "ip-blocked" + ? (DATACENTER_BLOCKED_REASONS[challengeType] ?? DEFAULT_DATACENTER_BLOCKED_REASON) + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout` + await reportBlocked(page, capture.blockedEvidence, { tier: 3, status, reason, statusCode: mainResponse.status }) + return { tier: 3, status, durationMs: Date.now() - start, reason } } // challengeWait calls waitForLoadState('load') but the CF interstitial iframe can @@ -199,6 +198,14 @@ export async function runTier3( const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() console.log(`[tier3] cloudflare-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason: "cloudflare-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "cloudflare-persistent" } } @@ -206,6 +213,14 @@ export async function runTier3( const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() console.log(`[tier3] imperva-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason: "imperva-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" } } @@ -213,6 +228,14 @@ export async function runTier3( const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() console.log(`[tier3] akamai-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason: "akamai-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "akamai-persistent" } } @@ -237,7 +260,16 @@ export async function runTier3( } if (isBlocked(mainResponse.status, html)) { - return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } + const reason = `http-${mainResponse.status}` + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason, + statusCode: mainResponse.status, + html, + screenshot: shot, + }) + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason } } const cookies: Cookie[] = toCookies(await freshCtx.cookies()) diff --git a/packages/tiers/src/tiers/4.ts b/packages/tiers/src/tiers/4.ts index e4818c0..65765cf 100644 --- a/packages/tiers/src/tiers/4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -3,6 +3,7 @@ import { closeTemporaryContext, FINGERPRINT, newFreshContext } from "@trawl/brow import type { CapturedResponseEntry, ConsoleLogEntry, Cookie, NetworkLogEntry, TierResult } from "@trawl/types" import { capturePageScreenshot } from "../screenshot" import { solvePageCaptchas } from "../solvers" +import { reportBlocked } from "../utils/blockedEvidence" import { attachPageCapture, type CaptureOptions } from "../utils/capture" import { routeChallengeWait } from "../utils/challengeRouter" import { snapshotChallengeCookies, toCookies } from "../utils/cookies" @@ -113,17 +114,15 @@ export async function runTier4( ) if (resolution !== "ok") { - return { - tier: 4, - status: resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout", - durationMs: Date.now() - start, - reason: - resolution === "captcha-required" - ? `${challengeType}-captcha-required` - : resolution === "ip-blocked" - ? "proxy-ip-blocked" - : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, - } + const status = resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout" + const reason = + resolution === "captcha-required" + ? `${challengeType}-captcha-required` + : resolution === "ip-blocked" + ? "proxy-ip-blocked" + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout` + await reportBlocked(page, capture.blockedEvidence, { tier: 4, status, reason, statusCode: mainResponse.status }) + return { tier: 4, status, durationMs: Date.now() - start, reason } } await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}) @@ -162,6 +161,14 @@ export async function runTier4( } if (isCloudflarePage(html, mainResponse.headers)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason: "cloudflare-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 4, status: "blocked", @@ -171,6 +178,14 @@ export async function runTier4( } if (hasImpervaChallenge(html)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason: "imperva-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 4, status: "blocked", @@ -180,6 +195,14 @@ export async function runTier4( } if (hasAkamaiChallenge(html)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason: "akamai-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 4, status: "blocked", @@ -209,7 +232,16 @@ export async function runTier4( } if (isBlocked(mainResponse.status, html)) { - return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } + const reason = `http-${mainResponse.status}` + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason, + statusCode: mainResponse.status, + html, + screenshot: shot, + }) + return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason } } const cookies: Cookie[] = toCookies(await proxyContext.cookies()) diff --git a/packages/tiers/src/utils/blockedEvidence.ts b/packages/tiers/src/utils/blockedEvidence.ts new file mode 100644 index 0000000..b4643ee --- /dev/null +++ b/packages/tiers/src/utils/blockedEvidence.ts @@ -0,0 +1,53 @@ +import type { BlockedEvidence, TierResult } from "@trawl/types" +import type { Page } from "patchright" +import { capturePageScreenshot } from "../screenshot" + +// The wall is the only artifact a blocked scrape has to hand back, and it is terminal-path +// data: a request that did not ask for it attaches nothing and reads nothing, and a capture +// that fails degrades the evidence rather than the outcome. One wall is kept per request, +// so the only unbounded dimension is the markup. +const MAX_HTML_CHARS = Number(process.env.BLOCKED_EVIDENCE_MAX_HTML_CHARS ?? 512_000) + +export interface BlockedEvidenceSink { + // Take an image of the wall too, on the branches that have not already taken one. + screenshot?: boolean + report(evidence: BlockedEvidence): void +} + +export interface BlockedOutcome { + tier: 2 | 3 | 4 + status: TierResult["status"] + reason?: string + statusCode?: number + // Markup and image the branch already holds; read from the page when absent. + html?: string + screenshot?: string +} + +export async function reportBlocked( + page: Page, + sink: BlockedEvidenceSink | undefined, + outcome: BlockedOutcome, +): Promise { + if (!sink) return + try { + const html = outcome.html ?? (await page.content().catch(() => undefined)) + // settle: false — a challenge wall never reaches network idle, so waiting for it only + // spends the budget the next tier still needs. + const screenshot = + outcome.screenshot ?? + (sink.screenshot ? await capturePageScreenshot(page, Number.POSITIVE_INFINITY, { settle: false }) : undefined) + sink.report({ + tier: outcome.tier, + status: outcome.status, + reason: outcome.reason, + url: page.url(), + statusCode: outcome.statusCode, + html: html?.slice(0, MAX_HTML_CHARS), + htmlTruncated: html !== undefined && html.length > MAX_HTML_CHARS ? true : undefined, + screenshot, + }) + } catch (err) { + console.log(`[blocked-evidence] capture failed: ${err instanceof Error ? err.message : String(err)}`) + } +} diff --git a/packages/tiers/src/utils/capture.ts b/packages/tiers/src/utils/capture.ts index c5b600c..c041a11 100644 --- a/packages/tiers/src/utils/capture.ts +++ b/packages/tiers/src/utils/capture.ts @@ -1,7 +1,7 @@ import type { CapturedResponseEntry, ConsoleLogEntry, NetworkLogEntry } from "@trawl/types" import type { ConsoleMessage, Page, Request } from "patchright" +import type { BlockedEvidenceSink } from "./blockedEvidence" import { captureLimit } from "./captureConfig" - import { attachResponseCapture, type ResponseCaptureOptions } from "./responseCapture" // Captured evidence lives in memory alongside a browser slot, so every dimension is @@ -29,6 +29,9 @@ export interface CaptureOptions extends ResponseCaptureOptions { consoleLogs?: boolean networkLogs?: boolean redirectChain?: boolean + // Where a tier hands back the challenge wall it could not clear. Attaches no listener + // and buffers nothing — the tier reads the page once, on the branch that gives up. + blockedEvidence?: BlockedEvidenceSink } export interface CapturedPageEvidence { diff --git a/packages/tiers/tests/blockedEvidence.test.ts b/packages/tiers/tests/blockedEvidence.test.ts new file mode 100644 index 0000000..d5f7198 --- /dev/null +++ b/packages/tiers/tests/blockedEvidence.test.ts @@ -0,0 +1,287 @@ +import { describe, expect, test } from "bun:test" +import type { BrowserHandle } from "@trawl/browser" +import type { BlockedEvidence, SessionData } from "@trawl/types" +import type { OrchestratorDeps } from "../src/orchestrator" +import { ScrapeError, scrape } from "../src/orchestrator" +import { runTier2 } from "../src/tiers/2" +import { runTier3 } from "../src/tiers/3" + +const WALL_HTML = `Access denied

403

${"blocked ".repeat(40)}` +const JPEG = Buffer.from("fake-jpeg-bytes") +const JPEG_BASE64 = JPEG.toString("base64") + +const fingerprint = { userAgent: "test-agent", platform: "Linux x86_64", locale: "en-US", timezone: "UTC" } +const session: SessionData = { cookies: [], userAgent: "cached-user-agent", savedAt: 1 } + +interface PageStub { + page: any + contentReads: () => number + screenshotCalls: () => number +} + +const makePage = (options: { html?: string; status?: number; contentThrows?: boolean } = {}): PageStub => { + const mainFrame = {} + let contentReads = 0 + let screenshotCalls = 0 + const navigationResponse = { + url: () => "https://example.com/blocked", + status: () => options.status ?? 403, + headers: () => ({}), + body: async () => Buffer.from(options.html ?? WALL_HTML), + request: () => ({ isNavigationRequest: () => true, frame: () => mainFrame }), + } + const page = { + url: () => "https://example.com/blocked", + title: async () => "Access denied", + content: async () => { + contentReads++ + if (options.contentThrows) throw new Error("Target page, context or browser has been closed") + return options.html ?? WALL_HTML + }, + goto: async () => {}, + on: (event: string, handler: (response: unknown) => void) => { + if (event === "response") handler(navigationResponse) + }, + off: () => {}, + once: () => {}, + mainFrame: () => mainFrame, + frames: () => [], + context: () => ({ cookies: async () => [] }), + evaluate: async () => "test-agent", + setExtraHTTPHeaders: async () => {}, + waitForLoadState: async () => {}, + close: async () => {}, + keyboard: { press: async () => {} }, + mouse: { move: async () => {}, click: async () => {} }, + screenshot: async () => { + screenshotCalls++ + return JPEG + }, + } + return { page, contentReads: () => contentReads, screenshotCalls: () => screenshotCalls } +} + +const poolHandle = (page: unknown): BrowserHandle => + ({ + id: 1, + lease: 1, + context: { newPage: async () => page, addCookies: async () => {}, cookies: async () => [] }, + browser: {}, + fingerprint, + }) satisfies BrowserHandle + +const freshHandle = (page: unknown): BrowserHandle => + ({ + id: 2, + lease: 1, + context: {}, + browser: { + newContext: async () => ({ + newPage: async () => page, + addInitScript: async () => {}, + cookies: async () => [], + close: async () => {}, + }), + }, + fingerprint, + }) satisfies BrowserHandle + +const sink = (screenshot?: boolean) => { + const reported: BlockedEvidence[] = [] + return { reported, sink: { screenshot, report: (evidence: BlockedEvidence) => reported.push(evidence) } } +} + +describe("blocked evidence", () => { + test("Tier 2 reports the wall it stopped at, with an image only when a screenshot was asked for", async () => { + const { reported, sink: withImage } = sink(true) + const imaged = makePage() + const blocked = await runTier2( + "https://example.com", + poolHandle(imaged.page), + session, + 4_000, + {}, + "GET", + "", + undefined, + true, + { blockedEvidence: withImage }, + ) + + expect(blocked.status).toBe("blocked") + expect(blocked.reason).toBe("http-403") + expect(reported).toHaveLength(1) + expect(reported[0]).toMatchObject({ + tier: 2, + status: "blocked", + reason: "http-403", + url: "https://example.com/blocked", + statusCode: 403, + }) + expect(reported[0].html).toContain("Access denied") + expect(reported[0].screenshot).toBe(JPEG_BASE64) + expect(reported[0].htmlTruncated).toBeUndefined() + + const { reported: textOnly, sink: withoutImage } = sink() + const unimaged = makePage() + await runTier2("https://example.com", poolHandle(unimaged.page), session, 4_000, {}, "GET", "", undefined, false, { + blockedEvidence: withoutImage, + }) + + expect(textOnly[0].html).toContain("Access denied") + expect(textOnly[0].screenshot).toBeUndefined() + expect(unimaged.screenshotCalls()).toBe(0) + }) + + test("a request that did not ask for evidence reads nothing extra off the wall", async () => { + const untouched = makePage() + const blocked = await runTier2("https://example.com", poolHandle(untouched.page), session, 4_000) + + expect(blocked.status).toBe("blocked") + expect(blocked.reason).toBe("http-403") + expect(untouched.screenshotCalls()).toBe(0) + expect(untouched.contentReads()).toBe(1) + }) + + test("Tier 3 reports the persistent wall and keeps the tier result unchanged", async () => { + const { reported, sink: asked } = sink(true) + const stub = makePage() + const blocked = await runTier3( + "https://example.com", + freshHandle(stub.page), + 4_000, + undefined, + {}, + "GET", + "", + undefined, + true, + { + blockedEvidence: asked, + }, + ) + + expect(blocked).toEqual({ tier: 3, status: "blocked", durationMs: blocked.durationMs, reason: "http-403" }) + expect(reported).toHaveLength(1) + expect(reported[0]).toMatchObject({ tier: 3, reason: "http-403", statusCode: 403 }) + expect(reported[0].screenshot).toBe(JPEG_BASE64) + }) + + test("markup past the cap is truncated and flagged rather than dropped", async () => { + const oversize = `Access denied${"x".repeat(600_000)}` + const { reported, sink: asked } = sink() + await runTier2( + "https://example.com", + poolHandle(makePage({ html: oversize }).page), + session, + 4_000, + {}, + "GET", + "", + undefined, + false, + { + blockedEvidence: asked, + }, + ) + + expect(reported[0].html).toHaveLength(512_000) + expect(reported[0].htmlTruncated).toBe(true) + }) + + test("a capture that fails degrades the evidence and never touches the tier's outcome", async () => { + const throwing = { + screenshot: true, + report: () => { + throw new Error("sink exploded") + }, + } + const blocked = await runTier2( + "https://example.com", + poolHandle(makePage().page), + session, + 4_000, + {}, + "GET", + "", + undefined, + true, + { blockedEvidence: throwing }, + ) + + expect(blocked.status).toBe("blocked") + expect(blocked.reason).toBe("http-403") + }) + + test("an unreadable page ends the tier as an error, with no wall to report", async () => { + const { reported, sink: asked } = sink(true) + const blocked = await runTier2( + "https://example.com", + poolHandle(makePage({ contentThrows: true }).page), + session, + 4_000, + {}, + "GET", + "", + undefined, + true, + { blockedEvidence: asked }, + ) + + expect(blocked.status).toBe("error") + expect(reported).toHaveLength(0) + }) +}) + +describe("blocked evidence through the orchestrator", () => { + const depsFor = (pool: unknown, fresh: unknown, hasSession = true): OrchestratorDeps => ({ + acquireBrowser: async () => ({ ...poolHandle(pool), browser: freshHandle(fresh).browser }), + releaseBrowser: () => {}, + loadSession: async () => (hasSession ? session : undefined), + saveSession: async () => {}, + invalidateSession: async () => {}, + }) + + test("the deepest tier that rendered a wall is the one that reaches the caller", async () => { + const wall = (marker: string) => + `Access denied

403

${marker}${"blocked ".repeat(40)}` + const tier2Wall = makePage({ html: wall("tier two wall") }) + const tier3Wall = makePage({ html: wall("tier three wall") }) + + const error = (await scrape( + { + url: "https://example.com", + skipHttp: true, + maxTier: 3, + maxTimeout: 4_000, + blockedEvidence: true, + screenshot: true, + }, + depsFor(tier2Wall.page, tier3Wall.page), + ).catch((err) => err)) as ScrapeError + + expect(error).toBeInstanceOf(ScrapeError) + expect(error.timings.map((t) => `${t.tier}:${t.status}:${t.reason}`)).toEqual([ + "2:blocked:http-403", + "3:blocked:http-403", + ]) + expect(error.blockedEvidence?.tier).toBe(3) + expect(error.blockedEvidence?.html).toContain("tier three wall") + expect(error.blockedEvidence?.screenshot).toBe(JPEG_BASE64) + }) + + test("no evidence rides the error unless the request asked for it", async () => { + const tier2Wall = makePage() + + const error = (await scrape( + { url: "https://example.com", skipHttp: true, maxTier: 2, maxTimeout: 4_000 }, + depsFor(tier2Wall.page, tier2Wall.page), + ).catch((err) => err)) as ScrapeError + + expect(error).toBeInstanceOf(ScrapeError) + expect(error.timings).toHaveLength(1) + expect(error.timings[0].reason).toBe("http-403") + expect(error.blockedEvidence).toBeUndefined() + expect(tier2Wall.screenshotCalls()).toBe(0) + }) +}) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 909f6be..93e2b39 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -52,6 +52,11 @@ export interface ScrapeRequest { // CSS selector that also ends the settle window early. Only meaningful alongside // `captureResponses`. waitForSelector?: string + // Opt-in evidence from a challenge wall no tier could clear. Costs nothing on a + // successful scrape: it is only ever attached to the terminal failure (`blockedEvidence` + // on the 500 body), never to `ScrapeResult`. The image rides along only when + // `screenshot` is also set. + blockedEvidence?: boolean } // One browser console message. Shaped after WebDriver's browser log so a consumer can @@ -94,6 +99,24 @@ export interface CapturedResponseEntry { error?: string } +// The challenge wall a scrape stopped at, from the last browser tier that rendered one. +// Returned only on the failure path (`blockedEvidence` on the 500 body) and only when the +// request asked for it — a blocked outcome is never dressed up as a successful result. +export interface BlockedEvidence { + tier: 2 | 3 | 4 + status: TierResult["status"] + // Same string as the matching `timings[].reason`, e.g. "cloudflare-persistent". + reason?: string + // Where the browser actually stood when it gave up, after any challenge redirects. + url: string + statusCode?: number + html?: string + // The wall's markup exceeded BLOCKED_EVIDENCE_MAX_HTML_CHARS and `html` is the head of it. + htmlTruncated?: boolean + // Base64 JPEG, present only when the request also asked for a `screenshot`. + screenshot?: string +} + export interface TierResult { tier: 1 | 2 | 3 | 4 status: "success" | "blocked" | "needs-js" | "timeout" | "error" | "skipped"