diff --git a/CHANGELOG.md b/CHANGELOG.md index 64456af..cbe9759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Optional blocked-outcome evidence: `blockedEvidence: true` on `POST /scrape` attaches the challenge wall the deepest browser tier stopped at — `html`, `url`, `statusCode`, `reason`, and the `screenshot` when one was requested — to the **500 body**, alongside the existing `error` and `timings`. A blocked scrape stays a failure and `ScrapeResult` never carries a wall as content (#53); this only makes the failure diagnosable and keeps the page as evidence. Off by default — without the flag nothing is read on the branch that gives up. Failures with no page to read (Tier 1, a context or page that never opened, a hard network failure, `about:neterror`, an empty document, pool exhaustion) carry `timings` only. The wall's markup is capped by `BLOCKED_EVIDENCE_MAX_HTML_CHARS` and flagged `htmlTruncated` past it, and a capture failure leaves the evidence off the body rather than changing the outcome. + ## [1.5.0] - 2026-09-04 ### Changed diff --git a/apps/api/src/routes/scrape.test.ts b/apps/api/src/routes/scrape.test.ts new file mode 100644 index 0000000..2c09bd8 --- /dev/null +++ b/apps/api/src/routes/scrape.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, test } from "bun:test" +import type { BrowserHandle } from "@trawl/browser" +import type { OrchestratorDeps } from "@trawl/tiers" +import type { SessionData } from "@trawl/types" +import { scrapeRoute } from "./scrape" + +const WALL_HTML = `Access denied

403

${"blocked ".repeat(40)}` +const JPEG_BASE64 = Buffer.from("fake-jpeg-bytes").toString("base64") + +const session: SessionData = { cookies: [], userAgent: "cached-user-agent", savedAt: 1 } + +const mainFrame = {} +const wallPage = { + url: () => "https://example.com/blocked", + title: async () => "Access denied", + content: async () => WALL_HTML, + goto: async () => {}, + on: (event: string, handler: (response: unknown) => void) => { + if (event !== "response") return + handler({ + url: () => "https://example.com/blocked", + status: () => 403, + headers: () => ({}), + body: async () => Buffer.from(WALL_HTML), + request: () => ({ isNavigationRequest: () => true, frame: () => mainFrame }), + }) + }, + off: () => {}, + once: () => {}, + mainFrame: () => mainFrame, + frames: () => [], + context: () => ({ cookies: async () => [] }), + evaluate: async () => "test-agent", + setExtraHTTPHeaders: async () => {}, + waitForLoadState: async () => {}, + close: async () => {}, + screenshot: async () => Buffer.from("fake-jpeg-bytes"), +} + +const blockedDeps = (): OrchestratorDeps => ({ + acquireBrowser: async () => + ({ + id: 1, + lease: 1, + headful: false, + context: { newPage: async () => wallPage, addCookies: async () => {}, cookies: async () => [] }, + browser: {}, + fingerprint: { userAgent: "test-agent", platform: "Linux x86_64", locale: "en-US", timezone: "UTC" }, + }) satisfies BrowserHandle, + releaseBrowser: () => {}, + loadSession: async () => session, + saveSession: async () => {}, + invalidateSession: async () => {}, +}) + +const post = (body: unknown) => + scrapeRoute(blockedDeps, () => ({})).handle( + new Request("http://localhost/scrape", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ) + +const blockedRequest = { url: "https://example.com", skipHttp: true, maxTier: 2, maxTimeout: 4_000 } + +describe("POST /scrape on a blocked outcome", () => { + test("still answers 500 with the per-tier attempt history", async () => { + const response = await post(blockedRequest) + + expect(response.status).toBe(500) + const body = await response.json() + expect(body.error).toContain("Max tier reached without success") + expect(body.timings).toEqual([{ tier: 2, status: "blocked", durationMs: expect.any(Number), reason: "http-403" }]) + expect(body.blockedEvidence).toBeUndefined() + }) + + test("carries the challenge wall when the request asked for it", async () => { + const response = await post({ ...blockedRequest, blockedEvidence: true, screenshot: true }) + + expect(response.status).toBe(500) + const body = await response.json() + expect(body.timings[0].reason).toBe("http-403") + expect(body.blockedEvidence).toMatchObject({ + tier: 2, + status: "blocked", + reason: "http-403", + url: "https://example.com/blocked", + statusCode: 403, + screenshot: JPEG_BASE64, + }) + expect(body.blockedEvidence.html).toContain("Access denied") + }) + + test("omits the image when only the markup was asked for", async () => { + const response = await post({ ...blockedRequest, blockedEvidence: true }) + + const body = await response.json() + expect(body.blockedEvidence.html).toContain("Access denied") + expect(body.blockedEvidence.screenshot).toBeUndefined() + }) +}) diff --git a/apps/api/src/routes/scrape.ts b/apps/api/src/routes/scrape.ts index 9a359f5..97b6384 100644 --- a/apps/api/src/routes/scrape.ts +++ b/apps/api/src/routes/scrape.ts @@ -1,4 +1,5 @@ import { PoolExhaustedError } from "@trawl/browser" +import type { OrchestratorDeps } from "@trawl/tiers" import { RequestValidationError, ScrapeError, sanitizeHeaders, scrape } from "@trawl/tiers" import type { ScrapeRequest } from "@trawl/types" import { Elysia } from "elysia" @@ -10,17 +11,17 @@ import { requestUrl, validateScrapeRequest } from "../validation" // Error mapping: // 503 — pool still initializing (native { error }) // 429 — pool exhausted (FlareSolverr envelope; uniform with /v1) -// 500 — other scrape exception (native { error }) -export function scrapeRoute() { +// 500 — other scrape exception (native { error, timings, blockedEvidence }) +export function scrapeRoute(deps: () => OrchestratorDeps = getDeps, poolReady: () => unknown = getPool) { return new Elysia().post("/scrape", async ({ body, set }) => { try { validateScrapeRequest(body) const req: ScrapeRequest = body - if (!getPool()) { + if (!poolReady()) { set.status = 503 return { error: "Browser pool initializing, retry in a few seconds" } } - return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, getDeps()) + return await scrape({ ...req, headers: sanitizeHeaders(req.headers) }, deps()) } catch (err) { if (err instanceof RequestValidationError) { set.status = err.statusCode @@ -32,7 +33,7 @@ export function scrapeRoute() { } set.status = 500 if (err instanceof ScrapeError) { - return { error: err.message, timings: err.timings } + return { error: err.message, timings: err.timings, blockedEvidence: err.blockedEvidence } } return { error: err instanceof Error ? err.message : String(err) } } diff --git a/apps/docs/api-reference/native-api.md b/apps/docs/api-reference/native-api.md index b53701f..7e9e4fe 100644 --- a/apps/docs/api-reference/native-api.md +++ b/apps/docs/api-reference/native-api.md @@ -25,6 +25,7 @@ interface ScrapeRequest { captureResponses?: string[] // URL patterns whose response bodies to capture, default none settleTimeout?: number // ms to wait after load for a match, default 15000 waitForSelector?: string // CSS selector that ends the settle window early + blockedEvidence?: boolean // return the challenge wall on the error, default false } ``` @@ -46,6 +47,7 @@ interface ScrapeRequest { | `captureResponses` | string[] | — | URL patterns — a substring, or a glob matched against the whole URL when the pattern contains `*` or `?` — whose response bodies are returned as `capturedResponses` (browser tiers 2–4) | | `settleTimeout` | number | 15000 | Milliseconds to hold the page open after load waiting for a match; ends early on the first captured body, on `waitForSelector`, or on network idle. Only read alongside `captureResponses` | | `waitForSelector` | string | — | CSS selector that also ends the settle window early. Only read alongside `captureResponses` | +| `blockedEvidence` | boolean | false | When no tier clears the challenge, attach the wall the last browser tier stopped at to the 500 body as `blockedEvidence`. It is never attached to a successful result — see the note below. The image rides along only when `screenshot` is also set | Captured response bodies, headers, console messages, and URLs can contain credentials, tokens, or personal data. Treat these opt-in diagnostic fields as sensitive output. @@ -108,6 +110,37 @@ interface TierResult { } ``` +## Blocked-Outcome Evidence + +A scrape that runs a browser but never clears the challenge is still a failure: it answers +**500**, and `ScrapeResult` never carries a challenge wall dressed up as content. + +`blockedEvidence: true` attaches the wall to that failure instead, so a caller can tell +"blocked by a challenge" from "TRAWL broke" and can keep the page as evidence: + +```typescript +interface BlockedEvidence { + tier: 2 | 3 | 4 // which browser tier rendered the wall + status: 'blocked' | 'timeout' + reason?: string // identical to the matching timings[].reason + url: string // where the browser stood, after any challenge redirects + statusCode?: number + html?: string // the wall's markup + htmlTruncated?: boolean // html is the head of a page over BLOCKED_EVIDENCE_MAX_HTML_CHARS + screenshot?: string // base64 JPEG, only when `screenshot` was also requested +} +``` + +The wall reported is the one from the **deepest** browser tier that rendered one — a Tier 3 +wall is replaced by Tier 4's when Tier 4 also fails. Some failures have no page to hand +back at all and carry no evidence: Tier 1 (a plain HTTP fetch, no browser), a tier that +could not open a context or a page, a hard network failure (DNS, connection refused, TLS), +an `about:neterror` page, an empty document, and a pool that was exhausted or still +initializing. In those cases `timings` alone tells the story. + +Reading the wall never changes the outcome: a capture failure leaves `blockedEvidence` off +the body and logs the reason, and the status code and `timings` are the same either way. + ## Examples ### Minimal request @@ -219,3 +252,27 @@ alone — no need to check server logs: ] } ``` + +When the request set `blockedEvidence: true` and a browser tier rendered a challenge wall, +the same body also carries that page — see +[Blocked-Outcome Evidence](#blocked-outcome-evidence): + +```json +{ + "error": "All tiers exhausted. Last failure: cloudflare-persistent", + "timings": [ + { "tier": 1, "status": "needs-js", "durationMs": 50, "reason": "cloudflare-challenge" }, + { "tier": 3, "status": "blocked", "durationMs": 2942, "reason": "cloudflare-persistent" }, + { "tier": 4, "status": "blocked", "durationMs": 7890, "reason": "cloudflare-persistent" } + ], + "blockedEvidence": { + "tier": 4, + "status": "blocked", + "reason": "cloudflare-persistent", + "url": "https://nowsecure.nl/", + "statusCode": 403, + "html": "Just a moment......", + "screenshot": "/9j/4AAQSkZJRgABAQAA..." + } +} +``` diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index e95cbdc..e4ed28e 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -258,6 +258,24 @@ bodies with a valid `Content-Length` are read. Compressed or unknown-size bodies returned with `body: null` and an error. Declared sizes are reserved cumulatively before reads start, so concurrent responses cannot exceed the total read budget. +## Blocked-Outcome Evidence + +Only read when a request sets `blockedEvidence: true` — see +[Native API](/api-reference/native-api#blocked-outcome-evidence). Without it no page is +read on the branch that gives up. One wall is kept per request, so its markup is the only +unbounded dimension. + +| Variable | Default | Purpose | +| --- | ---: | --- | +| `BLOCKED_EVIDENCE_MAX_HTML_CHARS` | `512000` | Characters of the wall kept; past it `html` is the head of the page and `htmlTruncated` is set | + +The wall is truncated rather than dropped: unlike a stylesheet or an image, the head of a +challenge page still carries the title, the vendor markers and the incident id a caller +classifies on. Reading it never fails the scrape — a capture failure leaves the evidence +off the error and logs the reason, and the HTTP status and `timings` are unchanged either +way. + + ## CAPTCHA audio and media tools TRAWL uses ffmpeg while solving supported CAPTCHA challenges. reCAPTCHA audio is converted before diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 81fd9c6..b687bba 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -1,6 +1,6 @@ import type { BrowserHandle } from "@trawl/browser" import { FINGERPRINT, FINGERPRINT_POOL } from "@trawl/browser" -import type { Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" +import type { BlockedEvidence, Cookie, ScrapeRequest, ScrapeResult, SessionData, TierResult } from "@trawl/types" import { runTier1 } from "./tiers/1" import { runTier2 } from "./tiers/2" import { runTier3 } from "./tiers/3" @@ -20,10 +20,15 @@ const MAX_PROXY_ATTEMPTS = 2 // wasn't reaching anyone outside the orchestrator. export class ScrapeError extends Error { timings: TierResult[] - constructor(message: string, timings: TierResult[]) { + // The challenge wall the last browser tier stopped at, when the caller asked for it. + // It rides the error rather than a `blocked` ScrapeResult on purpose: a wall is not a + // scrape, and callers keyed on the success shape must never be handed one. + blockedEvidence?: BlockedEvidence + constructor(message: string, timings: TierResult[], blockedEvidence?: BlockedEvidence) { super(message) this.name = "ScrapeError" this.timings = timings + this.blockedEvidence = blockedEvidence } } @@ -76,6 +81,10 @@ export async function scrape( const tier1Proxy = explicitProxy && /^https?:\/\//i.test(explicitProxy) ? explicitProxy : undefined const skipTier1ForProxy = Boolean(explicitProxy && !tier1Proxy) + // Evidence from the last browser tier that rendered a wall it could not clear. Kept out + // of `timings` — that stays the thin, machine-readable attempt history — and reached + // only via the thrown ScrapeError. + let blockedEvidence: BlockedEvidence | undefined const capture = { consoleLogs: req.consoleLogs, networkLogs: req.networkLogs, @@ -83,6 +92,14 @@ export async function scrape( captureResponses: req.captureResponses, settleTimeout: req.settleTimeout, waitForSelector: req.waitForSelector, + blockedEvidence: req.blockedEvidence + ? { + screenshot: req.screenshot, + report: (evidence: BlockedEvidence) => { + blockedEvidence = evidence + }, + } + : undefined, } const sanitizedHeaders = sanitizeHeaders(req.headers) @@ -146,7 +163,7 @@ export async function scrape( } if (maxTier < 2) { - throw new ScrapeError("Max tier reached without success", timings) + throw new ScrapeError("Max tier reached without success", timings, blockedEvidence) } // Acquire browser for tiers 2-4 @@ -232,7 +249,7 @@ export async function scrape( } if (maxTier < 3) { - throw new ScrapeError("Max tier reached without success", timings) + throw new ScrapeError("Max tier reached without success", timings, blockedEvidence) } // Tier 3: fresh challenge solve. Proxy resolves from (priority order) a per-request @@ -317,7 +334,7 @@ export async function scrape( } if (maxTier < 4) { - throw new ScrapeError("Max tier reached without success", timings) + throw new ScrapeError("Max tier reached without success", timings, blockedEvidence) } // Tier 4: residential proxy escalation — requires at least one residential proxy, @@ -327,6 +344,7 @@ export async function scrape( throw new ScrapeError( `Tier 3 failed (${t3.reason ?? t3.status}). Set RESIDENTIAL_PROXY_URL (or pass a proxy per-request) to enable Tier 4 proxy escalation.`, timings, + blockedEvidence, ) } @@ -403,7 +421,7 @@ export async function scrape( } } - throw new ScrapeError(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`, timings) + throw new ScrapeError(`All tiers exhausted. Last failure: ${t4.reason ?? t4.status}`, timings, blockedEvidence) } finally { if (!handleReleased) deps.releaseBrowser(handle) } diff --git a/packages/tiers/src/screenshot.ts b/packages/tiers/src/screenshot.ts index 3f3a058..240a15a 100644 --- a/packages/tiers/src/screenshot.ts +++ b/packages/tiers/src/screenshot.ts @@ -16,6 +16,7 @@ const MAX_BYTES = captureLimit(process.env.SCREENSHOT_MAX_BYTES, 4_000_000) export async function capturePageScreenshot( page: Page, budgetMs = Number.POSITIVE_INFINITY, + options: { settle?: boolean } = {}, ): Promise { const deadline = Date.now() + Math.max(budgetMs, 0) const remaining = (): number => Math.max(deadline - Date.now(), 0) @@ -23,11 +24,14 @@ export async function capturePageScreenshot( try { // The HTML is read the moment a challenge clears, before late content (images, // fonts, lazy hydration) has painted. Give the page a bounded chance to settle, - // then a short beat for whatever paints after the last request. + // then a short beat for whatever paints after the last request. A caller imaging a + // page that will never settle (a challenge wall) opts out of the wait. if (remaining() <= 0) return undefined - await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, remaining()) }).catch(() => {}) - const paintWaitMs = Math.min(300, remaining()) - if (paintWaitMs > 0) await new Promise((r) => setTimeout(r, paintWaitMs)) + if (options.settle !== false) { + await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, remaining()) }).catch(() => {}) + const paintWaitMs = Math.min(300, remaining()) + if (paintWaitMs > 0) await new Promise((r) => setTimeout(r, paintWaitMs)) + } const captureTimeout = Math.min(CAPTURE_TIMEOUT_MS, remaining()) if (captureTimeout <= 0) return undefined diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index 237506b..3a45af8 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -9,6 +9,7 @@ import type { } from "@trawl/types" import { capturePageScreenshot } from "../screenshot" import { solvePageCaptchas } from "../solvers" +import { reportBlocked } from "../utils/blockedEvidence" import { attachPageCapture, type CaptureOptions } from "../utils/capture" import { normalizeSameSite, toCookies } from "../utils/cookies" import { @@ -105,12 +106,26 @@ export async function runTier2( } if (isCloudflarePage(html, mainResponse.headers)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason: "session-expired", + statusCode: mainResponse.status, + html, + }) return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } // A cached session that lands back on Akamai's interstitial is stale — force a // fresh Tier-3 solve rather than returning the ~2KB challenge stub as content. if (hasAkamaiChallenge(html)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason: "akamai-session-expired", + statusCode: mainResponse.status, + html, + }) return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "akamai-session-expired" } } @@ -127,7 +142,15 @@ export async function runTier2( } if (isBlocked(mainResponse.status, html)) { - return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } + const reason = `http-${mainResponse.status}` + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason, + statusCode: mainResponse.status, + html, + }) + return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason } } // Attempt to solve any embedded captcha widgets (Turnstile, reCAPTCHA, hCaptcha). @@ -150,6 +173,14 @@ export async function runTier2( const finalHtml = await page.content() if (isCloudflarePage(finalHtml, mainResponse.headers)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 2, + status: "blocked", + reason: "session-expired", + statusCode: mainResponse.status, + html: finalHtml, + screenshot: shot, + }) return { tier: 2, status: "blocked", durationMs: Date.now() - start, reason: "session-expired" } } diff --git a/packages/tiers/src/tiers/3.ts b/packages/tiers/src/tiers/3.ts index 2800527..c731792 100644 --- a/packages/tiers/src/tiers/3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -3,6 +3,7 @@ import { closeTemporaryContext, FINGERPRINT, newFreshContext } from "@trawl/brow import type { CapturedResponseEntry, ConsoleLogEntry, Cookie, NetworkLogEntry, TierResult } from "@trawl/types" import { capturePageScreenshot } from "../screenshot" import { solvePageCaptchas } from "../solvers" +import { reportBlocked } from "../utils/blockedEvidence" import { attachPageCapture, type CaptureOptions } from "../utils/capture" import { routeChallengeWait } from "../utils/challengeRouter" import { snapshotChallengeCookies, toCookies } from "../utils/cookies" @@ -138,17 +139,15 @@ export async function runTier3( ) if (resolution !== "ok") { - return { - tier: 3, - status: resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout", - durationMs: Date.now() - start, - reason: - resolution === "captcha-required" - ? `${challengeType}-captcha-required` - : resolution === "ip-blocked" - ? (DATACENTER_BLOCKED_REASONS[challengeType] ?? DEFAULT_DATACENTER_BLOCKED_REASON) - : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, - } + const status = resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout" + const reason = + resolution === "captcha-required" + ? `${challengeType}-captcha-required` + : resolution === "ip-blocked" + ? (DATACENTER_BLOCKED_REASONS[challengeType] ?? DEFAULT_DATACENTER_BLOCKED_REASON) + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout` + await reportBlocked(page, capture.blockedEvidence, { tier: 3, status, reason, statusCode: mainResponse.status }) + return { tier: 3, status, durationMs: Date.now() - start, reason } } // challengeWait calls waitForLoadState('load') but the CF interstitial iframe can @@ -199,6 +198,14 @@ export async function runTier3( const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() console.log(`[tier3] cloudflare-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason: "cloudflare-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "cloudflare-persistent" } } @@ -206,6 +213,14 @@ export async function runTier3( const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() console.log(`[tier3] imperva-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason: "imperva-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "imperva-persistent" } } @@ -213,6 +228,14 @@ export async function runTier3( const pageTitle = await page.title().catch(() => "?") const pageUrl = page.url() console.log(`[tier3] akamai-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`) + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason: "akamai-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "akamai-persistent" } } @@ -237,7 +260,16 @@ export async function runTier3( } if (isBlocked(mainResponse.status, html)) { - return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } + const reason = `http-${mainResponse.status}` + await reportBlocked(page, capture.blockedEvidence, { + tier: 3, + status: "blocked", + reason, + statusCode: mainResponse.status, + html, + screenshot: shot, + }) + return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason } } const cookies: Cookie[] = toCookies(await freshCtx.cookies()) diff --git a/packages/tiers/src/tiers/4.ts b/packages/tiers/src/tiers/4.ts index e4818c0..65765cf 100644 --- a/packages/tiers/src/tiers/4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -3,6 +3,7 @@ import { closeTemporaryContext, FINGERPRINT, newFreshContext } from "@trawl/brow import type { CapturedResponseEntry, ConsoleLogEntry, Cookie, NetworkLogEntry, TierResult } from "@trawl/types" import { capturePageScreenshot } from "../screenshot" import { solvePageCaptchas } from "../solvers" +import { reportBlocked } from "../utils/blockedEvidence" import { attachPageCapture, type CaptureOptions } from "../utils/capture" import { routeChallengeWait } from "../utils/challengeRouter" import { snapshotChallengeCookies, toCookies } from "../utils/cookies" @@ -113,17 +114,15 @@ export async function runTier4( ) if (resolution !== "ok") { - return { - tier: 4, - status: resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout", - durationMs: Date.now() - start, - reason: - resolution === "captcha-required" - ? `${challengeType}-captcha-required` - : resolution === "ip-blocked" - ? "proxy-ip-blocked" - : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout`, - } + const status = resolution === "ip-blocked" || resolution === "captcha-required" ? "blocked" : "timeout" + const reason = + resolution === "captcha-required" + ? `${challengeType}-captcha-required` + : resolution === "ip-blocked" + ? "proxy-ip-blocked" + : `${challengeType === "none" ? "cloudflare" : challengeType}-challenge-timeout` + await reportBlocked(page, capture.blockedEvidence, { tier: 4, status, reason, statusCode: mainResponse.status }) + return { tier: 4, status, durationMs: Date.now() - start, reason } } await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}) @@ -162,6 +161,14 @@ export async function runTier4( } if (isCloudflarePage(html, mainResponse.headers)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason: "cloudflare-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 4, status: "blocked", @@ -171,6 +178,14 @@ export async function runTier4( } if (hasImpervaChallenge(html)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason: "imperva-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 4, status: "blocked", @@ -180,6 +195,14 @@ export async function runTier4( } if (hasAkamaiChallenge(html)) { + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason: "akamai-persistent", + statusCode: mainResponse.status, + html, + screenshot: shot, + }) return { tier: 4, status: "blocked", @@ -209,7 +232,16 @@ export async function runTier4( } if (isBlocked(mainResponse.status, html)) { - return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` } + const reason = `http-${mainResponse.status}` + await reportBlocked(page, capture.blockedEvidence, { + tier: 4, + status: "blocked", + reason, + statusCode: mainResponse.status, + html, + screenshot: shot, + }) + return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason } } const cookies: Cookie[] = toCookies(await proxyContext.cookies()) diff --git a/packages/tiers/src/utils/blockedEvidence.ts b/packages/tiers/src/utils/blockedEvidence.ts new file mode 100644 index 0000000..b4643ee --- /dev/null +++ b/packages/tiers/src/utils/blockedEvidence.ts @@ -0,0 +1,53 @@ +import type { BlockedEvidence, TierResult } from "@trawl/types" +import type { Page } from "patchright" +import { capturePageScreenshot } from "../screenshot" + +// The wall is the only artifact a blocked scrape has to hand back, and it is terminal-path +// data: a request that did not ask for it attaches nothing and reads nothing, and a capture +// that fails degrades the evidence rather than the outcome. One wall is kept per request, +// so the only unbounded dimension is the markup. +const MAX_HTML_CHARS = Number(process.env.BLOCKED_EVIDENCE_MAX_HTML_CHARS ?? 512_000) + +export interface BlockedEvidenceSink { + // Take an image of the wall too, on the branches that have not already taken one. + screenshot?: boolean + report(evidence: BlockedEvidence): void +} + +export interface BlockedOutcome { + tier: 2 | 3 | 4 + status: TierResult["status"] + reason?: string + statusCode?: number + // Markup and image the branch already holds; read from the page when absent. + html?: string + screenshot?: string +} + +export async function reportBlocked( + page: Page, + sink: BlockedEvidenceSink | undefined, + outcome: BlockedOutcome, +): Promise { + if (!sink) return + try { + const html = outcome.html ?? (await page.content().catch(() => undefined)) + // settle: false — a challenge wall never reaches network idle, so waiting for it only + // spends the budget the next tier still needs. + const screenshot = + outcome.screenshot ?? + (sink.screenshot ? await capturePageScreenshot(page, Number.POSITIVE_INFINITY, { settle: false }) : undefined) + sink.report({ + tier: outcome.tier, + status: outcome.status, + reason: outcome.reason, + url: page.url(), + statusCode: outcome.statusCode, + html: html?.slice(0, MAX_HTML_CHARS), + htmlTruncated: html !== undefined && html.length > MAX_HTML_CHARS ? true : undefined, + screenshot, + }) + } catch (err) { + console.log(`[blocked-evidence] capture failed: ${err instanceof Error ? err.message : String(err)}`) + } +} diff --git a/packages/tiers/src/utils/capture.ts b/packages/tiers/src/utils/capture.ts index c5b600c..c041a11 100644 --- a/packages/tiers/src/utils/capture.ts +++ b/packages/tiers/src/utils/capture.ts @@ -1,7 +1,7 @@ import type { CapturedResponseEntry, ConsoleLogEntry, NetworkLogEntry } from "@trawl/types" import type { ConsoleMessage, Page, Request } from "patchright" +import type { BlockedEvidenceSink } from "./blockedEvidence" import { captureLimit } from "./captureConfig" - import { attachResponseCapture, type ResponseCaptureOptions } from "./responseCapture" // Captured evidence lives in memory alongside a browser slot, so every dimension is @@ -29,6 +29,9 @@ export interface CaptureOptions extends ResponseCaptureOptions { consoleLogs?: boolean networkLogs?: boolean redirectChain?: boolean + // Where a tier hands back the challenge wall it could not clear. Attaches no listener + // and buffers nothing — the tier reads the page once, on the branch that gives up. + blockedEvidence?: BlockedEvidenceSink } export interface CapturedPageEvidence { diff --git a/packages/tiers/tests/blockedEvidence.test.ts b/packages/tiers/tests/blockedEvidence.test.ts new file mode 100644 index 0000000..d5f7198 --- /dev/null +++ b/packages/tiers/tests/blockedEvidence.test.ts @@ -0,0 +1,287 @@ +import { describe, expect, test } from "bun:test" +import type { BrowserHandle } from "@trawl/browser" +import type { BlockedEvidence, SessionData } from "@trawl/types" +import type { OrchestratorDeps } from "../src/orchestrator" +import { ScrapeError, scrape } from "../src/orchestrator" +import { runTier2 } from "../src/tiers/2" +import { runTier3 } from "../src/tiers/3" + +const WALL_HTML = `Access denied

403

${"blocked ".repeat(40)}` +const JPEG = Buffer.from("fake-jpeg-bytes") +const JPEG_BASE64 = JPEG.toString("base64") + +const fingerprint = { userAgent: "test-agent", platform: "Linux x86_64", locale: "en-US", timezone: "UTC" } +const session: SessionData = { cookies: [], userAgent: "cached-user-agent", savedAt: 1 } + +interface PageStub { + page: any + contentReads: () => number + screenshotCalls: () => number +} + +const makePage = (options: { html?: string; status?: number; contentThrows?: boolean } = {}): PageStub => { + const mainFrame = {} + let contentReads = 0 + let screenshotCalls = 0 + const navigationResponse = { + url: () => "https://example.com/blocked", + status: () => options.status ?? 403, + headers: () => ({}), + body: async () => Buffer.from(options.html ?? WALL_HTML), + request: () => ({ isNavigationRequest: () => true, frame: () => mainFrame }), + } + const page = { + url: () => "https://example.com/blocked", + title: async () => "Access denied", + content: async () => { + contentReads++ + if (options.contentThrows) throw new Error("Target page, context or browser has been closed") + return options.html ?? WALL_HTML + }, + goto: async () => {}, + on: (event: string, handler: (response: unknown) => void) => { + if (event === "response") handler(navigationResponse) + }, + off: () => {}, + once: () => {}, + mainFrame: () => mainFrame, + frames: () => [], + context: () => ({ cookies: async () => [] }), + evaluate: async () => "test-agent", + setExtraHTTPHeaders: async () => {}, + waitForLoadState: async () => {}, + close: async () => {}, + keyboard: { press: async () => {} }, + mouse: { move: async () => {}, click: async () => {} }, + screenshot: async () => { + screenshotCalls++ + return JPEG + }, + } + return { page, contentReads: () => contentReads, screenshotCalls: () => screenshotCalls } +} + +const poolHandle = (page: unknown): BrowserHandle => + ({ + id: 1, + lease: 1, + context: { newPage: async () => page, addCookies: async () => {}, cookies: async () => [] }, + browser: {}, + fingerprint, + }) satisfies BrowserHandle + +const freshHandle = (page: unknown): BrowserHandle => + ({ + id: 2, + lease: 1, + context: {}, + browser: { + newContext: async () => ({ + newPage: async () => page, + addInitScript: async () => {}, + cookies: async () => [], + close: async () => {}, + }), + }, + fingerprint, + }) satisfies BrowserHandle + +const sink = (screenshot?: boolean) => { + const reported: BlockedEvidence[] = [] + return { reported, sink: { screenshot, report: (evidence: BlockedEvidence) => reported.push(evidence) } } +} + +describe("blocked evidence", () => { + test("Tier 2 reports the wall it stopped at, with an image only when a screenshot was asked for", async () => { + const { reported, sink: withImage } = sink(true) + const imaged = makePage() + const blocked = await runTier2( + "https://example.com", + poolHandle(imaged.page), + session, + 4_000, + {}, + "GET", + "", + undefined, + true, + { blockedEvidence: withImage }, + ) + + expect(blocked.status).toBe("blocked") + expect(blocked.reason).toBe("http-403") + expect(reported).toHaveLength(1) + expect(reported[0]).toMatchObject({ + tier: 2, + status: "blocked", + reason: "http-403", + url: "https://example.com/blocked", + statusCode: 403, + }) + expect(reported[0].html).toContain("Access denied") + expect(reported[0].screenshot).toBe(JPEG_BASE64) + expect(reported[0].htmlTruncated).toBeUndefined() + + const { reported: textOnly, sink: withoutImage } = sink() + const unimaged = makePage() + await runTier2("https://example.com", poolHandle(unimaged.page), session, 4_000, {}, "GET", "", undefined, false, { + blockedEvidence: withoutImage, + }) + + expect(textOnly[0].html).toContain("Access denied") + expect(textOnly[0].screenshot).toBeUndefined() + expect(unimaged.screenshotCalls()).toBe(0) + }) + + test("a request that did not ask for evidence reads nothing extra off the wall", async () => { + const untouched = makePage() + const blocked = await runTier2("https://example.com", poolHandle(untouched.page), session, 4_000) + + expect(blocked.status).toBe("blocked") + expect(blocked.reason).toBe("http-403") + expect(untouched.screenshotCalls()).toBe(0) + expect(untouched.contentReads()).toBe(1) + }) + + test("Tier 3 reports the persistent wall and keeps the tier result unchanged", async () => { + const { reported, sink: asked } = sink(true) + const stub = makePage() + const blocked = await runTier3( + "https://example.com", + freshHandle(stub.page), + 4_000, + undefined, + {}, + "GET", + "", + undefined, + true, + { + blockedEvidence: asked, + }, + ) + + expect(blocked).toEqual({ tier: 3, status: "blocked", durationMs: blocked.durationMs, reason: "http-403" }) + expect(reported).toHaveLength(1) + expect(reported[0]).toMatchObject({ tier: 3, reason: "http-403", statusCode: 403 }) + expect(reported[0].screenshot).toBe(JPEG_BASE64) + }) + + test("markup past the cap is truncated and flagged rather than dropped", async () => { + const oversize = `Access denied${"x".repeat(600_000)}` + const { reported, sink: asked } = sink() + await runTier2( + "https://example.com", + poolHandle(makePage({ html: oversize }).page), + session, + 4_000, + {}, + "GET", + "", + undefined, + false, + { + blockedEvidence: asked, + }, + ) + + expect(reported[0].html).toHaveLength(512_000) + expect(reported[0].htmlTruncated).toBe(true) + }) + + test("a capture that fails degrades the evidence and never touches the tier's outcome", async () => { + const throwing = { + screenshot: true, + report: () => { + throw new Error("sink exploded") + }, + } + const blocked = await runTier2( + "https://example.com", + poolHandle(makePage().page), + session, + 4_000, + {}, + "GET", + "", + undefined, + true, + { blockedEvidence: throwing }, + ) + + expect(blocked.status).toBe("blocked") + expect(blocked.reason).toBe("http-403") + }) + + test("an unreadable page ends the tier as an error, with no wall to report", async () => { + const { reported, sink: asked } = sink(true) + const blocked = await runTier2( + "https://example.com", + poolHandle(makePage({ contentThrows: true }).page), + session, + 4_000, + {}, + "GET", + "", + undefined, + true, + { blockedEvidence: asked }, + ) + + expect(blocked.status).toBe("error") + expect(reported).toHaveLength(0) + }) +}) + +describe("blocked evidence through the orchestrator", () => { + const depsFor = (pool: unknown, fresh: unknown, hasSession = true): OrchestratorDeps => ({ + acquireBrowser: async () => ({ ...poolHandle(pool), browser: freshHandle(fresh).browser }), + releaseBrowser: () => {}, + loadSession: async () => (hasSession ? session : undefined), + saveSession: async () => {}, + invalidateSession: async () => {}, + }) + + test("the deepest tier that rendered a wall is the one that reaches the caller", async () => { + const wall = (marker: string) => + `Access denied

403

${marker}${"blocked ".repeat(40)}` + const tier2Wall = makePage({ html: wall("tier two wall") }) + const tier3Wall = makePage({ html: wall("tier three wall") }) + + const error = (await scrape( + { + url: "https://example.com", + skipHttp: true, + maxTier: 3, + maxTimeout: 4_000, + blockedEvidence: true, + screenshot: true, + }, + depsFor(tier2Wall.page, tier3Wall.page), + ).catch((err) => err)) as ScrapeError + + expect(error).toBeInstanceOf(ScrapeError) + expect(error.timings.map((t) => `${t.tier}:${t.status}:${t.reason}`)).toEqual([ + "2:blocked:http-403", + "3:blocked:http-403", + ]) + expect(error.blockedEvidence?.tier).toBe(3) + expect(error.blockedEvidence?.html).toContain("tier three wall") + expect(error.blockedEvidence?.screenshot).toBe(JPEG_BASE64) + }) + + test("no evidence rides the error unless the request asked for it", async () => { + const tier2Wall = makePage() + + const error = (await scrape( + { url: "https://example.com", skipHttp: true, maxTier: 2, maxTimeout: 4_000 }, + depsFor(tier2Wall.page, tier2Wall.page), + ).catch((err) => err)) as ScrapeError + + expect(error).toBeInstanceOf(ScrapeError) + expect(error.timings).toHaveLength(1) + expect(error.timings[0].reason).toBe("http-403") + expect(error.blockedEvidence).toBeUndefined() + expect(tier2Wall.screenshotCalls()).toBe(0) + }) +}) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 909f6be..93e2b39 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -52,6 +52,11 @@ export interface ScrapeRequest { // CSS selector that also ends the settle window early. Only meaningful alongside // `captureResponses`. waitForSelector?: string + // Opt-in evidence from a challenge wall no tier could clear. Costs nothing on a + // successful scrape: it is only ever attached to the terminal failure (`blockedEvidence` + // on the 500 body), never to `ScrapeResult`. The image rides along only when + // `screenshot` is also set. + blockedEvidence?: boolean } // One browser console message. Shaped after WebDriver's browser log so a consumer can @@ -94,6 +99,24 @@ export interface CapturedResponseEntry { error?: string } +// The challenge wall a scrape stopped at, from the last browser tier that rendered one. +// Returned only on the failure path (`blockedEvidence` on the 500 body) and only when the +// request asked for it — a blocked outcome is never dressed up as a successful result. +export interface BlockedEvidence { + tier: 2 | 3 | 4 + status: TierResult["status"] + // Same string as the matching `timings[].reason`, e.g. "cloudflare-persistent". + reason?: string + // Where the browser actually stood when it gave up, after any challenge redirects. + url: string + statusCode?: number + html?: string + // The wall's markup exceeded BLOCKED_EVIDENCE_MAX_HTML_CHARS and `html` is the head of it. + htmlTruncated?: boolean + // Base64 JPEG, present only when the request also asked for a `screenshot`. + screenshot?: string +} + export interface TierResult { tier: 1 | 2 | 3 | 4 status: "success" | "blocked" | "needs-js" | "timeout" | "error" | "skipped"