From 24002370e009ef0dddf872e9751d68d02def28f2 Mon Sep 17 00:00:00 2001 From: emandel2630 Date: Wed, 19 Aug 2026 12:23:50 -0400 Subject: [PATCH] feat(scrape): optional best-effort MHTML archive of the page `mhtml: true` on POST /scrape returns a multipart/related archive in `ScrapeResult.mhtml`: the rendered DOM first, then the stylesheets, scripts, images and fonts the browser tiers observed loading, each part carrying its own `Content-Location` so a reader can resolve it back to its URL. Text parts are quoted-printable over the raw bytes, binary parts base64, both wrapped at 76 characters, so the archive is pure 7-bit ASCII with CRLF endings and can be written straight to a .mhtml file. This is an assembled approximation, not an engine snapshot. Firefox exposes no equivalent of Chromium CDP's Page.captureSnapshot, so the archive is built from what the response listener saw: a resource served from the browser's own cache, fetched before the listener attached, or refused on read is simply absent, and nothing rewrites the document's URLs to point at the archived parts. A reader that resolves subresources by Content-Location (which is what browsers do with a saved MHTML) gets a usable page; a byte-faithful reproduction it is not. Off by default. Without the flag no subresource body is read and no listener is attached beyond the ones already there - a stock request is unchanged. The collection rides on the response listener #109 added rather than opening a second path, and unlike pattern capture it does not extend the page's lifetime: the archive takes what the page produced during its normal load. Bounds, all env-tunable and parsed with the same captureLimit as CAPTURE_*: MHTML_MAX_PARTS 200 subresources archived per page MHTML_MAX_PART_BYTES 2097152 bytes per subresource MHTML_MAX_TOTAL_CHARS 8388608 encoded chars across the archive MHTML_MAX_INFLIGHT_READS 32 bodies read at the same time MHTML_MAX_OMISSION_RECORDS 100 omissions listed by URL A part over its budget is dropped whole rather than trimmed - a truncated stylesheet or image is corrupt, not partial. A response is refused on its declared Content-Length before its body is ever read, so a burst of large subresources costs nothing to reject; an undeclared length is bounded by the in-flight read count and the post-read part cap. Every omission is counted in an `X-Trawl-Omitted-Resources` header and listed in a final text/plain part, so an archive that hits a cap is still a valid MHTML that says what it is missing. --- CHANGELOG.md | 3 + apps/docs/api-reference/native-api.md | 19 + apps/docs/getting-started/configuration.md | 25 + packages/tiers/src/orchestrator.ts | 4 + packages/tiers/src/tiers/2.ts | 2 + packages/tiers/src/tiers/3.ts | 2 + packages/tiers/src/tiers/4.ts | 2 + packages/tiers/src/utils/capture.ts | 9 +- packages/tiers/src/utils/mhtml.ts | 187 ++++++ packages/tiers/src/utils/responseCapture.ts | 128 +++- packages/tiers/tests/mhtml.test.ts | 638 ++++++++++++++++++++ packages/types/src/index.ts | 11 + 12 files changed, 1022 insertions(+), 8 deletions(-) create mode 100644 packages/tiers/src/utils/mhtml.ts create mode 100644 packages/tiers/tests/mhtml.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 64456af..f077450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Optional MHTML archive: `mhtml: true` on `POST /scrape` returns a `multipart/related` archive of the page in `ScrapeResult.mhtml` — the rendered document first, then the stylesheets, scripts, images and fonts observed loading, each with its own `Content-Location`. Assembled from the responses the browser tiers already see rather than snapshotted by the engine (Firefox has no `Page.captureSnapshot`), so it is an approximation of "Save as MHTML": a resource served from cache or fetched before the listener attached is absent. Off by default — without the flag no subresource body is read. Part count, per-part bytes, reads in flight and total archive size are bounded and tunable via `MHTML_*`; an oversize part is omitted whole rather than trimmed, and every omission is counted in an `X-Trawl-Omitted-Resources` header and listed in a final part so a capped archive is still a valid MHTML. + ## [1.5.0] - 2026-09-04 ### Changed diff --git a/apps/docs/api-reference/native-api.md b/apps/docs/api-reference/native-api.md index b53701f..fceaca8 100644 --- a/apps/docs/api-reference/native-api.md +++ b/apps/docs/api-reference/native-api.md @@ -25,6 +25,7 @@ interface ScrapeRequest { captureResponses?: string[] // URL patterns whose response bodies to capture, default none settleTimeout?: number // ms to wait after load for a match, default 15000 waitForSelector?: string // CSS selector that ends the settle window early + mhtml?: boolean // assemble an MHTML archive of the page, default false } ``` @@ -46,6 +47,7 @@ interface ScrapeRequest { | `captureResponses` | string[] | — | URL patterns — a substring, or a glob matched against the whole URL when the pattern contains `*` or `?` — whose response bodies are returned as `capturedResponses` (browser tiers 2–4) | | `settleTimeout` | number | 15000 | Milliseconds to hold the page open after load waiting for a match; ends early on the first captured body, on `waitForSelector`, or on network idle. Only read alongside `captureResponses` | | `waitForSelector` | string | — | CSS selector that also ends the settle window early. Only read alongside `captureResponses` | +| `mhtml` | boolean | false | Assemble a `multipart/related` MHTML archive of the page on the browser tiers (2–4) and return it as `mhtml`. An approximation of "Save as MHTML", not an engine snapshot — see the note below | Captured response bodies, headers, console messages, and URLs can contain credentials, tokens, or personal data. Treat these opt-in diagnostic fields as sensitive output. @@ -70,6 +72,7 @@ interface ScrapeResult { networkLogs?: NetworkLogEntry[] // resource timings, same presence rules as consoleLogs redirectChain?: string[] // URLs the main document walked, same presence rules as consoleLogs capturedResponses?: CapturedResponseEntry[] // matched response bodies, [] when nothing matched + mhtml?: string // multipart/related archive of the page, only when requested and a browser tier served the page } interface ConsoleLogEntry { @@ -108,6 +111,22 @@ interface TierResult { } ``` +## MHTML Archives + +`mhtml: true` returns a single `multipart/related` document: the rendered DOM first, then +the stylesheets, scripts, images and fonts that were observed loading. It is pure 7-bit +ASCII with CRLF line endings, so it can be written straight to a `.mhtml` file, and every +part carries a `Content-Location` so a reader can resolve it back to its URL. + +It is an **assembled approximation, not an engine snapshot.** Firefox exposes no +equivalent of Chromium's `Page.captureSnapshot`, so the archive is built from what the +response listener saw: a resource the browser served from its own cache, fetched before +the listener attached, or refused to hand over is absent. Anything dropped for a byte +budget is counted in the `X-Trawl-Omitted-Resources` header and listed in a final +`text/plain` part, so an archive that hits a cap is still a valid MHTML that says what it +is missing. Bounds are tunable via `MHTML_*` — see +[Configuration](/getting-started/configuration#mhtml-archives). + ## Examples ### Minimal request diff --git a/apps/docs/getting-started/configuration.md b/apps/docs/getting-started/configuration.md index e95cbdc..571adc9 100644 --- a/apps/docs/getting-started/configuration.md +++ b/apps/docs/getting-started/configuration.md @@ -258,6 +258,31 @@ bodies with a valid `Content-Length` are read. Compressed or unknown-size bodies returned with `body: null` and an error. Declared sizes are reserved cumulatively before reads start, so concurrent responses cannot exceed the total read budget. +## MHTML Archives + +Only read when a request sets `mhtml: true` — see +[Native API](/api-reference/native-api#mhtml-archives). Without it no subresource body is +read. The archive keeps many small parts rather than a few large bodies, so it carries +budgets of its own rather than sharing the response-body ones. + +| Variable | Default | Purpose | +| --- | ---: | --- | +| `MHTML_MAX_PARTS` | `200` | Subresources archived per page | +| `MHTML_MAX_PART_BYTES` | `2097152` | Bytes per subresource; a larger one is omitted whole | +| `MHTML_MAX_TOTAL_CHARS` | `8388608` | Encoded characters across all subresources of one page — the size of the archive itself | +| `MHTML_MAX_INFLIGHT_READS` | `32` | Subresource bodies read at the same time; a burst past this is omitted rather than held | +| `MHTML_MAX_OMISSION_RECORDS` | `100` | Omissions listed by URL in the archive; the rest are only counted | + +Bodies are read as they arrive, so a page whose subresources all complete at once is +bounded twice over: by `MHTML_MAX_INFLIGHT_READS`, and by the archive budget itself +wherever the response declared a `Content-Length`. At the defaults an archiving request +holds at most the archive (8 MiB) plus its reads in flight. + +A subresource is omitted rather than trimmed — a truncated stylesheet or image is corrupt, +not partial. Every omission is counted in the archive's `X-Trawl-Omitted-Resources` header +and listed in its final part, so an archive that hits a cap is still valid and still says +what is missing. An assembly failure leaves `mhtml` unset and never fails the scrape. + ## CAPTCHA audio and media tools TRAWL uses ffmpeg while solving supported CAPTCHA challenges. reCAPTCHA audio is converted before diff --git a/packages/tiers/src/orchestrator.ts b/packages/tiers/src/orchestrator.ts index 81fd9c6..40be001 100644 --- a/packages/tiers/src/orchestrator.ts +++ b/packages/tiers/src/orchestrator.ts @@ -83,6 +83,7 @@ export async function scrape( captureResponses: req.captureResponses, settleTimeout: req.settleTimeout, waitForSelector: req.waitForSelector, + mhtml: req.mhtml, } const sanitizedHeaders = sanitizeHeaders(req.headers) @@ -225,6 +226,7 @@ export async function scrape( networkLogs: t2.networkLogs, redirectChain: t2.redirectChain, capturedResponses: t2.capturedResponses, + mhtml: t2.mhtml, } } // Session failed — purge it @@ -313,6 +315,7 @@ export async function scrape( networkLogs: t3.networkLogs, redirectChain: t3.redirectChain, capturedResponses: t3.capturedResponses, + mhtml: t3.mhtml, } } @@ -400,6 +403,7 @@ export async function scrape( networkLogs: t4.networkLogs, redirectChain: t4.redirectChain, capturedResponses: t4.capturedResponses, + mhtml: t4.mhtml, } } diff --git a/packages/tiers/src/tiers/2.ts b/packages/tiers/src/tiers/2.ts index 237506b..cde125b 100644 --- a/packages/tiers/src/tiers/2.ts +++ b/packages/tiers/src/tiers/2.ts @@ -41,6 +41,7 @@ export interface Tier2Result extends TierResult { networkLogs?: NetworkLogEntry[] redirectChain?: string[] capturedResponses?: CapturedResponseEntry[] + mhtml?: string } export async function runTier2( @@ -172,6 +173,7 @@ export async function runTier2( screenshot: shot, ...evidence, redirectChain: capture.redirectChain ? mainResponse.redirectChain : undefined, + mhtml: pageCapture.archive(page.url(), finalHtml), } } catch (err) { return { diff --git a/packages/tiers/src/tiers/3.ts b/packages/tiers/src/tiers/3.ts index 2800527..3081db8 100644 --- a/packages/tiers/src/tiers/3.ts +++ b/packages/tiers/src/tiers/3.ts @@ -57,6 +57,7 @@ export interface Tier3Result extends TierResult { networkLogs?: NetworkLogEntry[] redirectChain?: string[] capturedResponses?: CapturedResponseEntry[] + mhtml?: string } export async function runTier3( @@ -258,6 +259,7 @@ export async function runTier3( screenshot: shot, ...evidence, redirectChain: capture.redirectChain ? mainResponse.redirectChain : undefined, + mhtml: pageCapture.archive(page.url(), html), } } catch (err) { return { diff --git a/packages/tiers/src/tiers/4.ts b/packages/tiers/src/tiers/4.ts index e4818c0..54db56a 100644 --- a/packages/tiers/src/tiers/4.ts +++ b/packages/tiers/src/tiers/4.ts @@ -41,6 +41,7 @@ export interface Tier4Result extends TierResult { networkLogs?: NetworkLogEntry[] redirectChain?: string[] capturedResponses?: CapturedResponseEntry[] + mhtml?: string } export async function runTier4( @@ -230,6 +231,7 @@ export async function runTier4( screenshot: shot, ...evidence, redirectChain: capture.redirectChain ? mainResponse.redirectChain : undefined, + mhtml: pageCapture.archive(page.url(), html), } } catch (err) { return { diff --git a/packages/tiers/src/utils/capture.ts b/packages/tiers/src/utils/capture.ts index c5b600c..eccd51d 100644 --- a/packages/tiers/src/utils/capture.ts +++ b/packages/tiers/src/utils/capture.ts @@ -41,9 +41,11 @@ export interface PageCapture { /** Holds the page open for the response-capture settle window; a no-op otherwise. */ settle(budgetMs: number): Promise drain(budgetMs?: number): Promise + /** Multipart/related archive of the observed subresources; undefined unless asked for. */ + archive(url: string, html: string): string | undefined } -const NO_CAPTURE: PageCapture = { settle: async () => {}, drain: async () => ({}) } +const NO_CAPTURE: PageCapture = { settle: async () => {}, drain: async () => ({}), archive: () => undefined } const ms = (value: number): number => Math.round(value * 100) / 100 @@ -53,13 +55,15 @@ const ms = (value: number): number => Math.round(value * 100) / 100 * the caller — a capture failure degrades that field, not the scrape. */ export function attachPageCapture(page: Page, options: CaptureOptions): PageCapture { - if (!options.consoleLogs && !options.networkLogs && !options.captureResponses?.length) return NO_CAPTURE + if (!options.consoleLogs && !options.networkLogs && !options.captureResponses?.length && !options.mhtml) + return NO_CAPTURE const responses = attachResponseCapture(page, options) if (!options.consoleLogs && !options.networkLogs) { return { settle: (budgetMs) => responses.settle(budgetMs), drain: async (budgetMs) => ({ capturedResponses: await responses.drain(budgetMs) }), + archive: (url, html) => responses.archive(url, html), } } @@ -154,6 +158,7 @@ export function attachPageCapture(page: Page, options: CaptureOptions): PageCapt return { settle: (budgetMs) => responses.settle(budgetMs), + archive: (url, html) => responses.archive(url, html), async drain(budgetMs = SIZES_TIMEOUT_MS) { const drainStarted = Date.now() const totalBudget = Math.max(0, Number.isFinite(budgetMs) ? budgetMs : 0) diff --git a/packages/tiers/src/utils/mhtml.ts b/packages/tiers/src/utils/mhtml.ts new file mode 100644 index 0000000..24e0113 --- /dev/null +++ b/packages/tiers/src/utils/mhtml.ts @@ -0,0 +1,187 @@ +import { randomBytes } from "node:crypto" + +// Firefox exposes no equivalent of Chromium's Page.captureSnapshot, so the archive is +// assembled from the subresources the response listener already observed rather than +// serialized by the engine. It is a valid multipart/related document, not a byte-faithful +// snapshot: anything the browser served from cache, fetched before the listener attached, +// or refused to hand over is absent, and the omission part records what is missing. +const ARCHIVED_RESOURCE_TYPES = new Set(["document", "stylesheet", "script", "image", "font"]) + +const DEFAULT_CONTENT_TYPES: Record = { + document: "text/html", + stylesheet: "text/css", + script: "application/javascript", + image: "application/octet-stream", + font: "application/octet-stream", +} + +// Longest line a quoted-printable or base64 part may use, per RFC 2045. +const MAX_LINE_CHARS = 76 + +// A header line is not folded, so a Content-Location longer than this is not archivable — +// folding a URL is what makes an MHTML unreadable in the browsers that reject it. +export const MAX_LOCATION_CHARS = 2_000 + +export type MhtmlOmissionReason = + | "over-part-budget" + | "archive-budget-exhausted" + | "part-count-cap" + | "read-slots-busy" + | "body-read-failed" + | "location-too-long" + +export interface MhtmlPart { + location: string + contentType: string + encoding: "quoted-printable" | "base64" + content: string +} + +export interface MhtmlOmission { + location: string + reason: MhtmlOmissionReason +} + +export const isArchivableResourceType = (resourceType: string): boolean => ARCHIVED_RESOURCE_TYPES.has(resourceType) + +export const defaultContentType = (resourceType: string): string => + DEFAULT_CONTENT_TYPES[resourceType] ?? "application/octet-stream" + +/** Strips anything a header line cannot carry — a line break above all. */ +const headerSafe = (value: string, maxChars = MAX_LOCATION_CHARS): string => { + let safe = "" + for (const char of value) { + const code = char.codePointAt(0) ?? 0 + safe += code >= 0x20 && code <= 0x7e ? char : encodeURIComponent(char) + if (safe.length >= maxChars) break + } + return safe.slice(0, maxChars) +} + +const wrap = (value: string): string => { + const lines: string[] = [] + for (let at = 0; at < value.length; at += MAX_LINE_CHARS) lines.push(value.slice(at, at + MAX_LINE_CHARS)) + return lines.join("\r\n") +} + +const HEX = "0123456789ABCDEF" + +const escaped = (byte: number): string => `=${HEX[byte >> 4]}${HEX[byte & 0xf]}` + +/** + * RFC 2045 quoted-printable, over the raw bytes rather than a decoded string, so a part + * whose charset is not UTF-8 stays byte-faithful under its own declared charset. + */ +export const toQuotedPrintable = (raw: Buffer): string => { + const lines: string[] = [] + let line = "" + + const push = (token: string) => { + if (line.length + token.length > MAX_LINE_CHARS - 1) { + lines.push(`${line}=`) + line = "" + } + line += token + } + + const breakLine = () => { + // Trailing whitespace would be eaten by a transport that rewraps lines. + const last = line.at(-1) + if (last === " " || last === "\t") { + line = line.slice(0, -1) + push(escaped(last === " " ? 0x20 : 0x09)) + } + lines.push(line) + line = "" + } + + for (let at = 0; at < raw.length; at++) { + const byte = raw[at] + if (byte === 0x0d && raw[at + 1] === 0x0a) { + at++ + breakLine() + } else if (byte === 0x0a || byte === 0x0d) { + breakLine() + } else if (byte === 0x3d || byte < 0x20 || byte > 0x7e) { + push(escaped(byte)) + } else { + push(String.fromCharCode(byte)) + } + } + + breakLine() + return lines.join("\r\n") +} + +export const encodePart = (raw: Buffer, asText: boolean): Pick => + asText + ? { encoding: "quoted-printable", content: toQuotedPrintable(raw) } + : { encoding: "base64", content: wrap(raw.toString("base64")) } + +const omissionPart = (omissions: MhtmlOmission[], omitted: number): string => { + const listed = omissions.map((o) => `${o.reason} ${headerSafe(o.location)}`).join("\r\n") + const unlisted = omitted > omissions.length ? `\r\n(${omitted - omissions.length} further omissions not listed)` : "" + return `${omitted} resource(s) omitted from this archive.\r\n${listed}${unlisted}` +} + +export interface MhtmlDocument { + url: string + html: string + parts: MhtmlPart[] + omissions: MhtmlOmission[] + omitted: number +} + +/** + * Assembles one multipart/related archive with the main document first. The boundary is + * checked against every part so no content can terminate the archive early. + */ +export function assembleMhtml(document: MhtmlDocument): string { + const main: MhtmlPart = { + location: document.url, + contentType: "text/html; charset=utf-8", + ...encodePart(Buffer.from(document.html, "utf8"), true), + } + const parts = [main, ...document.parts.filter((part) => part.location !== document.url)] + if (document.omitted > 0) { + parts.push({ + location: "", + contentType: "text/plain; charset=utf-8", + ...encodePart(Buffer.from(omissionPart(document.omissions, document.omitted), "utf8"), true), + }) + } + + let boundary = "" + for (let attempt = 0; attempt < 4; attempt++) { + boundary = `----MultipartBoundary--trawl${randomBytes(12).toString("hex")}----` + if (!parts.some((part) => part.content.includes(boundary))) break + } + + const header = [ + "From: ", + `Snapshot-Content-Location: ${headerSafe(document.url)}`, + `Date: ${new Date().toUTCString()}`, + "MIME-Version: 1.0", + "X-Trawl-Archive: assembled-from-observed-subresources", + ...(document.omitted > 0 ? [`X-Trawl-Omitted-Resources: ${document.omitted}`] : []), + `Content-Type: multipart/related; type="text/html"; boundary="${boundary}"`, + ].join("\r\n") + + const body = parts + .map((part) => + [ + `--${boundary}`, + `Content-Type: ${headerSafe(part.contentType, 200)}`, + `Content-Transfer-Encoding: ${part.encoding}`, + part.location + ? `Content-Location: ${headerSafe(part.location)}` + : "Content-ID: ", + "", + part.content, + "", + ].join("\r\n"), + ) + .join("") + + return `${header}\r\n\r\n${body}--${boundary}--\r\n` +} diff --git a/packages/tiers/src/utils/responseCapture.ts b/packages/tiers/src/utils/responseCapture.ts index 4042a58..5b72b6f 100644 --- a/packages/tiers/src/utils/responseCapture.ts +++ b/packages/tiers/src/utils/responseCapture.ts @@ -1,6 +1,9 @@ import type { CapturedResponseEntry } from "@trawl/types" import type { Page, Response } from "patchright" import { captureLimit } from "./captureConfig" + +import type { MhtmlOmission, MhtmlOmissionReason, MhtmlPart } from "./mhtml" +import { assembleMhtml, defaultContentType, encodePart, isArchivableResourceType, MAX_LOCATION_CHARS } from "./mhtml" import { isTextContentType } from "./response" const MAX_PATTERNS = captureLimit(process.env.CAPTURE_MAX_PATTERNS, 10) @@ -14,6 +17,20 @@ const MAX_SETTLE_MS = captureLimit(process.env.CAPTURE_MAX_SETTLE_MS, 60_000) const IDLE_FLOOR_MS = captureLimit(process.env.CAPTURE_IDLE_FLOOR_MS, 5_000) const MAX_STRING_CHARS = captureLimit(process.env.CAPTURE_MAX_METADATA_CHARS, 2_000) +// The MHTML archive keeps many small parts rather than a few large bodies, so it carries +// budgets of its own instead of sharing the pattern-capture ones. A part over its own +// budget is dropped whole — a trimmed stylesheet or image is corrupt, not partial — and +// every drop is recorded in the archive itself. The total is counted in encoded characters +// rather than raw bytes, because that is what the archive costs to hold and to return. +const MAX_ARCHIVE_PARTS = captureLimit(process.env.MHTML_MAX_PARTS, 200) +const MAX_ARCHIVE_PART_BYTES = captureLimit(process.env.MHTML_MAX_PART_BYTES, 2_097_152) +const MAX_ARCHIVE_TOTAL_CHARS = captureLimit(process.env.MHTML_MAX_TOTAL_CHARS, 8_388_608) +const MAX_ARCHIVE_OMISSION_RECORDS = captureLimit(process.env.MHTML_MAX_OMISSION_RECORDS, 100) +// Bodies are read as they arrive, so a page whose subresources all complete at once would +// otherwise hold every one of them at the same time. Reads in flight are bounded by count, +// and by the archive budget itself where the response declared its length. +const MAX_ARCHIVE_INFLIGHT_READS = captureLimit(process.env.MHTML_MAX_INFLIGHT_READS, 32) + const NEVER = new Promise(() => {}) const COMPRESSED_ENCODINGS = new Set(["gzip", "br", "deflate", "zstd"]) @@ -21,13 +38,20 @@ export interface ResponseCaptureOptions { captureResponses?: string[] settleTimeout?: number waitForSelector?: string + mhtml?: boolean } export interface ResponseCapture { settle(budgetMs: number): Promise drain(budgetMs?: number): Promise + /** Assembles the archived subresources around the main document. Call after drain(). */ + archive(url: string, html: string): string | undefined } -const NO_RESPONSE_CAPTURE: ResponseCapture = { settle: async () => {}, drain: async () => undefined } +const NO_RESPONSE_CAPTURE: ResponseCapture = { + settle: async () => {}, + drain: async () => undefined, + archive: () => undefined, +} const message = (err: unknown): string => (err instanceof Error ? err.message : String(err)) const bounded = (value: string): string => value.slice(0, MAX_STRING_CHARS) @@ -80,15 +104,30 @@ const compilePatterns = (patterns: string[]): Array<(url: string) => boolean> => const snapshot = (entries: CapturedResponseEntry[]): CapturedResponseEntry[] => entries.map((entry) => ({ ...entry, headers: { ...entry.headers } })) +/** + * Records the bodies of the responses whose URL matches a caller-supplied pattern, and + * the archivable subresources when an MHTML archive was asked for — both off one response + * listener. Attaches nothing at all unless one of them was asked for, detaches on drain + * and again on page close, and never throws into the caller: a body that cannot be read + * carries its own `error`, an archive part that cannot be read is recorded as omitted. + */ export function attachResponseCapture(page: Page, options: ResponseCaptureOptions): ResponseCapture { - const patterns = options.captureResponses - if (!Array.isArray(patterns) || patterns.length === 0) return NO_RESPONSE_CAPTURE - const matchers = compilePatterns(patterns) - if (matchers.length === 0) return NO_RESPONSE_CAPTURE + const patterns = Array.isArray(options.captureResponses) ? options.captureResponses : [] + const matchers = patterns.length > 0 ? compilePatterns(patterns) : [] + const archiving = Boolean(options.mhtml) + if (matchers.length === 0 && !archiving) return NO_RESPONSE_CAPTURE const entries: CapturedResponseEntry[] = [] const pending: Promise[] = [] const timers = new Set>() + const parts: MhtmlPart[] = [] + const archived = new Set() + const omissions: MhtmlOmission[] = [] + let archiveCharsUsed = 0 + let archiveReads = 0 + let archiveInflightReads = 0 + let archiveInflightBytes = 0 + let omitted = 0 let storedBytes = 0 let reservedReadBytes = 0 let accepting = true @@ -130,8 +169,73 @@ export function attachResponseCapture(page: Page, options: ResponseCaptureOption } } + const omit = (location: string, reason: MhtmlOmissionReason): void => { + omitted++ + if (omissions.length < MAX_ARCHIVE_OMISSION_RECORDS) omissions.push({ location, reason }) + } + + const readArchivePart = async ( + response: Response, + url: string, + resourceType: string, + declared: number, + ): Promise => { + try { + const raw = Buffer.from(await response.body()) + if (raw.length > MAX_ARCHIVE_PART_BYTES) return omit(url, "over-part-budget") + const contentType = response.headers()["content-type"] ?? defaultContentType(resourceType) + const encoded = encodePart(raw, isTextContentType(contentType)) + if (archiveCharsUsed + encoded.content.length > MAX_ARCHIVE_TOTAL_CHARS) { + return omit(url, "archive-budget-exhausted") + } + archiveCharsUsed += encoded.content.length + parts.push({ location: url, contentType, ...encoded }) + } catch { + // A retry of the same URL may still succeed, so it keeps its place in the queue. + archived.delete(url) + omit(url, "body-read-failed") + } finally { + archiveInflightReads-- + archiveInflightBytes -= declared + } + } + + // Only the resources a browser needs to render the page offline are archived; an XHR + // payload or a media stream would bloat the archive without changing what it shows. + const collectArchivePart = (response: Response) => { + let url = "?" + try { + url = response.url() + const status = response.status() + // A redirect has no body of its own, and an error page is not the resource. + if (status >= 300) return + const resourceType = response.request().resourceType() + if (!isArchivableResourceType(resourceType) || archived.has(url)) return + if (url.length > MAX_LOCATION_CHARS) return omit(url, "location-too-long") + if (archiveReads >= MAX_ARCHIVE_PARTS) return omit(url, "part-count-cap") + if (archiveInflightReads >= MAX_ARCHIVE_INFLIGHT_READS) return omit(url, "read-slots-busy") + // A declared length past the part cap skips the read entirely; the real length is + // checked after the read, and the in-flight read count bounds what an undeclared one + // can cost. + const declared = declaredLength(response) ?? 0 + if (declared > MAX_ARCHIVE_PART_BYTES) return omit(url, "over-part-budget") + if (archiveCharsUsed + archiveInflightBytes + declared > MAX_ARCHIVE_TOTAL_CHARS) { + return omit(url, "archive-budget-exhausted") + } + archiveReads++ + archiveInflightReads++ + archiveInflightBytes += declared + archived.add(url) + pending.push(readArchivePart(response, url, resourceType, declared)) + } catch { + omit(url, "body-read-failed") + } + } + const onResponse = (response: Response) => { if (!accepting) return + if (archiving) collectArchivePart(response) + if (matchers.length === 0) return try { const rawUrl = response.url() if (!matchers.some((matches) => matches(rawUrl))) return @@ -190,6 +294,8 @@ export function attachResponseCapture(page: Page, options: ResponseCaptureOption return { async settle(budgetMs) { + // The archive rides on the page's existing lifetime; only pattern capture extends it. + if (matchers.length === 0) return const requested = options.settleTimeout ?? SETTLE_MS const windowMs = Math.max(0, Math.min(requested, MAX_SETTLE_MS, Number.isFinite(budgetMs) ? budgetMs : 0)) if (windowMs === 0) return @@ -220,7 +326,17 @@ export function attachResponseCapture(page: Page, options: ResponseCaptureOption if (dropped > 0) console.log(`[capture] dropped ${dropped} matched responses past the configured caps`) for (const timer of timers) clearTimeout(timer) timers.clear() - return snapshot(entries) + return matchers.length > 0 ? snapshot(entries) : undefined + }, + archive(url, html) { + if (!archiving) return undefined + try { + if (omitted > 0) console.log(`[capture] omitted ${omitted} resources from the mhtml archive`) + return assembleMhtml({ url, html, parts, omissions, omitted }) + } catch (err) { + console.log(`[capture] mhtml assembly failed: ${message(err)}`) + return undefined + } }, } } diff --git a/packages/tiers/tests/mhtml.test.ts b/packages/tiers/tests/mhtml.test.ts new file mode 100644 index 0000000..282ce3f --- /dev/null +++ b/packages/tiers/tests/mhtml.test.ts @@ -0,0 +1,638 @@ +import { describe, expect, test } from "bun:test" +import type { BrowserHandle } from "@trawl/browser" +import type { SessionData } from "@trawl/types" +import type { OrchestratorDeps } from "../src/orchestrator" +import { scrape } from "../src/orchestrator" +import { runTier2 } from "../src/tiers/2" +import { assembleMhtml, toQuotedPrintable } from "../src/utils/mhtml" +import { attachResponseCapture } from "../src/utils/responseCapture" + +const PAGE_HTML = `Shell${"content ".repeat(20)}` + +const session: SessionData = { cookies: [], userAgent: "cached-user-agent", savedAt: 1 } + +const fingerprint = { userAgent: "test-agent", platform: "Linux x86_64", locale: "en-US", timezone: "UTC" } + +const mainFrame = {} + +const resource = ( + url: string, + resourceType: string, + options: { status?: number; contentType?: string | null; body?: Buffer | Error; contentLength?: number } = {}, +) => ({ + url: () => url, + status: () => options.status ?? 200, + headers: () => ({ + ...(options.contentType === null ? {} : { "content-type": options.contentType ?? "text/css" }), + ...(options.contentLength ? { "content-length": String(options.contentLength) } : {}), + }), + body: async () => { + if (options.body instanceof Error) throw options.body + return options.body ?? Buffer.from("body{color:red}") + }, + request: () => ({ isNavigationRequest: () => false, frame: () => mainFrame, resourceType: () => resourceType }), +}) + +const documentResponse = (url: string) => ({ + ...resource(url, "document", { contentType: "text/html", body: Buffer.from("origin") }), + request: () => ({ isNavigationRequest: () => true, frame: () => mainFrame, resourceType: () => "document" }), +}) + +interface Emitter { + emit(event: string, arg: unknown): void + listeners(event: string): number +} + +const makeEmitter = (): Emitter & { on: unknown; off: unknown; once: unknown } => { + const handlers = new Map void>>() + return { + on: (event: string, handler: (arg: never) => void) => { + handlers.set(event, [...(handlers.get(event) ?? []), handler]) + }, + once: (event: string, handler: (arg: never) => void) => { + handlers.set(event, [...(handlers.get(event) ?? []), handler]) + }, + off: (event: string, handler: (arg: never) => void) => { + handlers.set( + event, + (handlers.get(event) ?? []).filter((h) => h !== handler), + ) + }, + emit(event, arg) { + for (const handler of [...(handlers.get(event) ?? [])]) handler(arg as never) + }, + listeners: (event: string) => (handlers.get(event) ?? []).length, + } +} + +const makePage = (onGoto: (emit: Emitter["emit"]) => void = () => {}) => { + const emitter = makeEmitter() + const page = { + ...emitter, + url: () => "https://example.com/landed", + title: async () => "Shell", + content: async () => PAGE_HTML, + goto: async () => { + onGoto(emitter.emit) + }, + mainFrame: () => mainFrame, + frames: () => [], + context: () => ({ cookies: async () => [] }), + evaluate: async () => "test-agent", + setExtraHTTPHeaders: async () => {}, + waitForLoadState: async () => {}, + waitForSelector: async () => { + throw new Error("selector timeout") + }, + screenshot: async () => Buffer.from("jpeg"), + close: async () => {}, + } + return { page, emitter } +} + +const poolHandle = (page: unknown): BrowserHandle => + ({ + id: 1, + lease: 1, + context: { newPage: async () => page, addCookies: async () => {}, cookies: async () => [] }, + browser: {}, + fingerprint, + }) satisfies BrowserHandle + +/** Splits an archive into its header block and its parts, the way a MIME reader would. */ +const parseArchive = (archive: string) => { + const [header, ...rest] = archive.split("\r\n\r\n") + const boundary = /boundary="([^"]+)"/.exec(header)?.[1] ?? "" + const body = rest.join("\r\n\r\n") + expect(body.endsWith(`--${boundary}--\r\n`)).toBe(true) + const parts = body + .slice(0, -`--${boundary}--\r\n`.length) + .split(`--${boundary}\r\n`) + .filter((chunk) => chunk.length > 0) + .map((chunk) => { + const split = chunk.indexOf("\r\n\r\n") + const headers = Object.fromEntries( + chunk + .slice(0, split) + .split("\r\n") + .map((line) => { + const at = line.indexOf(": ") + return [line.slice(0, at).toLowerCase(), line.slice(at + 2)] + }), + ) + return { headers, content: chunk.slice(split + 4).replace(/\r\n$/, "") } + }) + const headers = Object.fromEntries( + header.split("\r\n").map((line) => { + const at = line.indexOf(": ") + return [line.slice(0, at).toLowerCase(), line.slice(at + 2)] + }), + ) + return { boundary, headers, parts } +} + +const fromQuotedPrintable = (content: string): Buffer => + Buffer.from( + content + .replace(/=\r\n/g, "") + .replace(/\r\n/g, "\n") + .replace(/=([0-9A-F]{2})/g, (_, hex) => String.fromCharCode(Number.parseInt(hex, 16))), + "latin1", + ) + +describe("quoted-printable", () => { + test("round-trips non-ASCII bytes and escapes the escape character", () => { + const raw = Buffer.from("café = 1 ✓\r\nsecond line\r\n", "utf8") + const encoded = toQuotedPrintable(raw) + + expect(encoded).toContain("=3D") + expect(encoded).not.toContain("café") + expect(fromQuotedPrintable(encoded).equals(Buffer.from(raw.toString("utf8").replace(/\r\n/g, "\n")))).toBe(true) + }) + + test("keeps every line inside the 76-character limit", () => { + const encoded = toQuotedPrintable(Buffer.from(`${"x".repeat(500)}\r\n${"é".repeat(200)}`, "utf8")) + + for (const line of encoded.split("\r\n")) expect(line.length).toBeLessThanOrEqual(76) + }) + + test("encodes whitespace that would otherwise end a line", () => { + const encoded = toQuotedPrintable(Buffer.from("trailing \r\ntab\t\r\n", "utf8")) + + expect(encoded.split("\r\n")[0]).toBe("trailing=20") + expect(encoded.split("\r\n")[1]).toBe("tab=09") + expect(toQuotedPrintable(Buffer.from("no newline ", "utf8"))).toBe("no newline=20") + }) +}) + +describe("assembleMhtml", () => { + test("puts the main document first, under multipart/related", () => { + const archive = assembleMhtml({ + url: "https://example.com/page", + html: "main", + parts: [ + { + location: "https://example.com/app.css", + contentType: "text/css", + encoding: "quoted-printable", + content: "body{}", + }, + ], + omissions: [], + omitted: 0, + }) + + const { headers, parts } = parseArchive(archive) + + expect(headers["content-type"]).toContain("multipart/related") + expect(headers["content-type"]).toContain('type="text/html"') + expect(headers["snapshot-content-location"]).toBe("https://example.com/page") + expect(headers["mime-version"]).toBe("1.0") + expect(headers["x-trawl-archive"]).toBe("assembled-from-observed-subresources") + expect(parts).toHaveLength(2) + expect(parts[0].headers["content-type"]).toBe("text/html; charset=utf-8") + expect(parts[0].headers["content-location"]).toBe("https://example.com/page") + expect(fromQuotedPrintable(parts[0].content).toString("utf8")).toBe("main") + expect(parts[1].headers["content-location"]).toBe("https://example.com/app.css") + }) + + test("drops a subresource part that duplicates the main document", () => { + const archive = assembleMhtml({ + url: "https://example.com/page", + html: "", + parts: [ + { + location: "https://example.com/page", + contentType: "text/html", + encoding: "quoted-printable", + content: "stale", + }, + ], + omissions: [], + omitted: 0, + }) + + expect(parseArchive(archive).parts).toHaveLength(1) + }) + + test("stays valid when parts were omitted, and says what is missing", () => { + const archive = assembleMhtml({ + url: "https://example.com/page", + html: "", + parts: [], + omissions: [{ location: "https://example.com/huge.png", reason: "over-part-budget" }], + omitted: 3, + }) + + const { headers, parts } = parseArchive(archive) + + expect(headers["x-trawl-omitted-resources"]).toBe("3") + expect(parts).toHaveLength(2) + expect(parts[1].headers["content-id"]).toBe("") + const note = fromQuotedPrintable(parts[1].content).toString("utf8") + expect(note).toContain("3 resource(s) omitted") + expect(note).toContain("over-part-budget https://example.com/huge.png") + expect(note).toContain("(2 further omissions not listed)") + }) + + test("cannot be talked into injecting a header line", () => { + const archive = assembleMhtml({ + url: "https://example.com/a\r\nX-Injected: yes", + html: "", + parts: [], + omissions: [], + omitted: 0, + }) + + expect(archive.split("\r\n").some((line) => line.startsWith("X-Injected"))).toBe(false) + expect(parseArchive(archive).headers["snapshot-content-location"]).toBe( + "https://example.com/a%0D%0AX-Injected: yes", + ) + }) + + test("emits pure ASCII, so the archive survives being written as text", () => { + const archive = assembleMhtml({ + url: "https://example.com/páge", + html: "café ✓", + parts: [ + { + location: "https://example.com/f.woff", + contentType: "font/woff", + encoding: "base64", + content: "AAECAw==", + }, + ], + omissions: [], + omitted: 0, + }) + + expect([...archive].every((char) => (char.codePointAt(0) ?? 0) < 0x80)).toBe(true) + }) +}) + +describe("subresource archiving", () => { + test("attaches nothing and archives nothing without the flag", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, {}) + emitter.emit("response", resource("https://example.com/app.css", "stylesheet")) + + expect(emitter.listeners("response")).toBe(0) + expect(await capture.drain()).toBeUndefined() + expect(capture.archive("https://example.com/", PAGE_HTML)).toBeUndefined() + }) + + test("archives the renderable resource types and skips the rest", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true }) + emitter.emit("response", resource("https://example.com/app.css", "stylesheet")) + emitter.emit("response", resource("https://example.com/app.js", "script", { contentType: "text/javascript" })) + emitter.emit( + "response", + resource("https://example.com/logo.png", "image", { contentType: "image/png", body: Buffer.from([1, 2, 3, 4]) }), + ) + emitter.emit("response", resource("https://example.com/f.woff2", "font", { contentType: "font/woff2" })) + emitter.emit("response", resource("https://example.com/frame.html", "document", { contentType: "text/html" })) + emitter.emit("response", resource("https://example.com/api/items", "xhr", { contentType: "application/json" })) + emitter.emit("response", resource("https://example.com/clip.mp4", "media", { contentType: "video/mp4" })) + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(headers["x-trawl-omitted-resources"]).toBeUndefined() + expect(parts.map((part) => part.headers["content-location"])).toEqual([ + "https://example.com/", + "https://example.com/app.css", + "https://example.com/app.js", + "https://example.com/logo.png", + "https://example.com/f.woff2", + "https://example.com/frame.html", + ]) + expect(parts[1].headers["content-transfer-encoding"]).toBe("quoted-printable") + expect(parts[3].headers["content-transfer-encoding"]).toBe("base64") + expect(Buffer.from(parts[3].content, "base64").equals(Buffer.from([1, 2, 3, 4]))).toBe(true) + }) + + test("archives a resource once however often the page refetches it", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true }) + emitter.emit("response", resource("https://example.com/app.css", "stylesheet")) + emitter.emit("response", resource("https://example.com/app.css", "stylesheet")) + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(parts).toHaveLength(2) + expect(headers["x-trawl-omitted-resources"]).toBeUndefined() + }) + + test("skips redirects and error responses", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true }) + emitter.emit("response", resource("https://example.com/moved.css", "stylesheet", { status: 302 })) + emitter.emit("response", resource("https://example.com/gone.css", "stylesheet", { status: 404 })) + await capture.drain() + + expect(parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string).parts).toHaveLength(1) + }) + + test("omits a part over the per-part budget on its declared length alone", async () => { + const { page, emitter } = makePage() + let read = false + + const capture = attachResponseCapture(page as never, { mhtml: true }) + const huge = resource("https://example.com/huge.css", "stylesheet", { contentLength: 3_000_000 }) + emitter.emit("response", { + ...huge, + body: async () => { + read = true + return Buffer.alloc(0) + }, + }) + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(read).toBe(false) + expect(headers["x-trawl-omitted-resources"]).toBe("1") + expect(fromQuotedPrintable(parts[1].content).toString("utf8")).toContain( + "over-part-budget https://example.com/huge.css", + ) + }) + + test("omits a part whose real length overruns the budget", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true }) + emitter.emit( + "response", + resource("https://example.com/huge.css", "stylesheet", { body: Buffer.alloc(2_097_153, 0x61) }), + ) + await capture.drain() + + const { headers } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(headers["x-trawl-omitted-resources"]).toBe("1") + }) + + test("stops archiving once the whole-archive budget is spent", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true }) + for (let n = 0; n < 4; n++) { + emitter.emit( + "response", + resource(`https://example.com/${n}.png`, "image", { + contentType: "image/png", + body: Buffer.alloc(2_097_152, n), + }), + ) + } + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(Number(headers["x-trawl-omitted-resources"])).toBeGreaterThan(0) + expect(parts.length).toBeGreaterThan(1) + expect(fromQuotedPrintable(parts.at(-1)?.content as string).toString("utf8")).toContain( + "archive-budget-exhausted https://example.com/", + ) + }) + + test("bounds how many subresource reads are in flight at once", async () => { + const { page, emitter } = makePage() + let release = () => {} + const gate = new Promise((resolve) => { + release = resolve + }) + + const capture = attachResponseCapture(page as never, { mhtml: true }) + for (let n = 0; n < 34; n++) { + emitter.emit("response", { + ...resource(`https://example.com/${n}.css`, "stylesheet"), + body: async () => { + await gate + return Buffer.from("body{color:red}") + }, + }) + } + release() + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(parts).toHaveLength(34) + expect(headers["x-trawl-omitted-resources"]).toBe("2") + expect(fromQuotedPrintable(parts.at(-1)?.content as string).toString("utf8")).toContain( + "read-slots-busy https://example.com/32.css", + ) + }) + + test("refuses a burst on its declared lengths, before any body is read", async () => { + const { page, emitter } = makePage() + const reads: string[] = [] + let release = () => {} + const gate = new Promise((resolve) => { + release = resolve + }) + + const capture = attachResponseCapture(page as never, { mhtml: true }) + for (let n = 0; n < 5; n++) { + const url = `https://example.com/${n}.png` + emitter.emit("response", { + ...resource(url, "image", { contentType: "image/png", contentLength: 2_097_152 }), + body: async () => { + reads.push(url) + await gate + return Buffer.alloc(16, n) + }, + }) + } + release() + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(reads).toHaveLength(4) + expect(headers["x-trawl-omitted-resources"]).toBe("1") + expect(fromQuotedPrintable(parts.at(-1)?.content as string).toString("utf8")).toContain( + "archive-budget-exhausted https://example.com/4.png", + ) + }) + + test("archives a response whose content-length is unusable without unsettling the budget", async () => { + const { page, emitter } = makePage() + let release = () => {} + const gate = new Promise((resolve) => { + release = resolve + }) + + const capture = attachResponseCapture(page as never, { mhtml: true }) + emitter.emit("response", { + ...resource("https://example.com/a.css", "stylesheet"), + headers: () => ({ "content-type": "text/css", "content-length": "chunked" }), + }) + for (let n = 0; n < 5; n++) { + emitter.emit("response", { + ...resource(`https://example.com/${n}.png`, "image", { contentType: "image/png", contentLength: 2_097_152 }), + body: async () => { + await gate + return Buffer.alloc(16, n) + }, + }) + } + release() + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(parts.map((part) => part.headers["content-location"])).toContain("https://example.com/a.css") + expect(headers["x-trawl-omitted-resources"]).toBe("1") + }) + + test("records a part whose body cannot be read", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true }) + emitter.emit( + "response", + resource("https://example.com/app.css", "stylesheet", { body: new Error("target closed") }), + ) + await capture.drain() + + const { headers, parts } = parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string) + + expect(headers["x-trawl-omitted-resources"]).toBe("1") + expect(fromQuotedPrintable(parts[1].content).toString("utf8")).toContain( + "body-read-failed https://example.com/app.css", + ) + }) + + test("survives a response object that throws", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true }) + emitter.emit("response", { + url: () => { + throw new Error("response gone") + }, + }) + await capture.drain() + + expect( + parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string).headers["x-trawl-omitted-resources"], + ).toBe("1") + }) + + test("leaves capturedResponses absent and never holds the page open", async () => { + const { page } = makePage() + const capture = attachResponseCapture(page as never, { mhtml: true }) + + const started = Date.now() + await capture.settle(30_000) + + expect(Date.now() - started).toBeLessThan(500) + expect(await capture.drain()).toBeUndefined() + }) + + test("archives alongside pattern capture off one listener", async () => { + const { page, emitter } = makePage() + + const capture = attachResponseCapture(page as never, { mhtml: true, captureResponses: ["/api/items"] }) + emitter.emit("response", resource("https://example.com/app.css", "stylesheet")) + emitter.emit("response", resource("https://example.com/api/items", "xhr", { contentType: "application/json" })) + + expect(emitter.listeners("response")).toBe(1) + + const entries = await capture.drain() + + expect(entries?.map((entry) => entry.url)).toEqual(["https://example.com/api/items"]) + expect(parseArchive(capture.archive("https://example.com/", PAGE_HTML) as string).parts).toHaveLength(2) + }) +}) + +describe("browser tiers", () => { + const shell = (emit: Emitter["emit"]) => { + emit("response", documentResponse("https://example.com/landed")) + emit("response", resource("https://example.com/app.css", "stylesheet")) + } + + test("Tier 2 returns an archive only when asked", async () => { + const requested = makePage(shell) + const withArchive = await runTier2( + "https://example.com", + poolHandle(requested.page), + session, + 4_000, + {}, + "GET", + "", + undefined, + false, + { mhtml: true }, + ) + + expect(withArchive.status).toBe("success") + const { parts } = parseArchive(withArchive.mhtml as string) + // The main document response is dropped in favour of the rendered DOM at the same URL. + expect(parts.map((part) => part.headers["content-location"])).toEqual([ + "https://example.com/landed", + "https://example.com/app.css", + ]) + expect(fromQuotedPrintable(parts[0].content).toString("utf8")).toBe(PAGE_HTML) + expect(withArchive.capturedResponses).toBeUndefined() + + const untouched = makePage(shell) + const withoutArchive = await runTier2("https://example.com", poolHandle(untouched.page), session, 4_000) + + expect(withoutArchive.status).toBe("success") + expect(withoutArchive.mhtml).toBeUndefined() + // Only the main-document tracker's listener — the archive attached nothing. + expect(untouched.emitter.listeners("response")).toBe(1) + expect(requested.emitter.listeners("response")).toBe(1) + }) +}) + +describe("orchestrator", () => { + const depsFor = (page: unknown): OrchestratorDeps => ({ + acquireBrowser: async () => poolHandle(page), + releaseBrowser: () => {}, + loadSession: async () => session, + saveSession: async () => {}, + invalidateSession: async () => {}, + }) + + test("passes the flag through and returns the archive", async () => { + const { page } = makePage((emit) => { + emit("response", documentResponse("https://example.com/landed")) + emit("response", resource("https://example.com/app.css", "stylesheet")) + }) + + const result = await scrape( + { url: "https://example.com", skipHttp: true, maxTier: 2, maxTimeout: 4_000, mhtml: true }, + depsFor(page), + ) + + expect(result.tier).toBe(2) + expect(result.mhtml).toContain("multipart/related") + expect(result.mhtml).toContain("Content-Location: https://example.com/app.css") + }) + + test("omits the field entirely by default", async () => { + const { page, emitter } = makePage((emit) => { + emit("response", resource("https://example.com/app.css", "stylesheet")) + }) + + const result = await scrape( + { url: "https://example.com", skipHttp: true, maxTier: 2, maxTimeout: 4_000 }, + depsFor(page), + ) + + expect(result.tier).toBe(2) + expect(result.mhtml).toBeUndefined() + expect(emitter.listeners("response")).toBe(1) + }) +}) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 909f6be..2846976 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -52,6 +52,11 @@ export interface ScrapeRequest { // CSS selector that also ends the settle window early. Only meaningful alongside // `captureResponses`. waitForSelector?: string + // Opt-in MHTML archive of the page from the browser tiers (2-4), returned as + // `ScrapeResult.mhtml`. Assembled from the subresources the response listener observes, + // not snapshotted by the engine — Firefox has no Page.captureSnapshot. Off by default: + // it reads every archivable subresource body. + mhtml?: boolean } // One browser console message. Shaped after WebDriver's browser log so a consumer can @@ -139,6 +144,12 @@ export interface ScrapeResult { // Present (possibly empty, meaning nothing matched) only when the request asked for // capture and a browser tier served the page. capturedResponses?: CapturedResponseEntry[] + // Multipart/related archive of the page: the rendered document first, then the CSS, + // script, image and font subresources that were observed loading. Same presence rules + // as `consoleLogs`. An approximation of a browser "Save as MHTML", not a byte-faithful + // snapshot — resources served from cache are absent, and anything dropped for a byte + // budget is listed in the archive's own final part. + mhtml?: string } export interface SessionData {