From 520849aea1605d5df0e3aafeaa2d99d57486078f Mon Sep 17 00:00:00 2001 From: nathan nelson Date: Tue, 11 Aug 2026 13:35:23 -0600 Subject: [PATCH] =?UTF-8?q?fix(guards):=20close=20the=20silent-omission=20?= =?UTF-8?q?gaps=20=E2=80=94=20crawl,=20extract,=20money,=20charset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every defect here fails SILENTLY, which is the class that actually costs research: the tool returns something that looks clean while the guard that should have warned could not arm. crawl ran extraction alone — no looks_blocked, no assess_extraction. A consent banner or bot wall was written to disk as a numbered page file, cached, and listed in INDEX.md with an ordinary token count. Walls are now refused: never written, never cached, links never followed, reported as `refused: ()` with a count in the summary, and page warnings ride in the index too. This matters more on crawl than anywhere else because its output is read back off disk later instead of being re-fetched — a wall written once is a wall trusted for weeks. extract cached whatever it fetched into the SAME pages.html row scrape reads, so one call could poison both verbs, and it returned the wall's own structured data — usually "no structured data found", which reads as "this page has none" rather than "we never saw the page". Money matching had four blind spots, each disarming the dropped-price guard on the exact page class it exists for. Figures are now canonicalised symbol-first and space-free through money_figures(), because `$ 49` and `$49` are one price: - `$49`, ordinary pricing-grid markup — tag stripping inserts a space and the pattern required a digit right after the symbol, so it matched NOTHING on such a page - `$49,` in prose kept the comma, so the same figure compared unequal against its table rendering and the guard reported a FALSE drop - `19,99 €`, the German/French suffix form on exactly the tarif/preise pages the title path targets, never matched at all - `$1.1 billion` matched as `$1`, silently changing the figure `tarif` had no word boundary, so English "tariffs" matched and handed news articles the low arming floor the design reserves for pricing pages — and tariff coverage is the genre that scatters real dollar figures through prose. Guard thresholds are now link-invariant: they are absolute character counts, but scrape passes the caller's include_links through, and link syntax inflates markdown ~70% — enough for a failing page to go silent because the caller asked for links. html_to_text is charset-aware instead of utf-8-only. This was pinned as a cosmetic `raw` limitation, which understated it: this function produces the page_text that ARMS the price/stub/gated guards, so a latin-1 pricing page had its £ figures replaced before counting and no guard could arm. fresh=True now evicts a poisoned row — replace_poisoned was only reachable through the cache-read branch fresh skips, so the caller got good content while the cache kept the wall for the rest of the TTL and the next scrape replayed it. CALIBRATION: replaying all 194 cached pages gives ZERO guard-verdict changes, so nothing above widened or narrowed what fires on the real corpus. Distinct figures went 363 -> 377, every delta verified legitimate. NOT FIXED, deliberately: the collapsed-column guard still has no real-page coverage. A replacement signal was built and measured and REJECTED — it separates the fixtures (quo 19, heyrosie 1) then fires just as hard on federal prose (FAR 52.227-20 19, artificialintelligenceact.eu 16, NASA SBIR 13). No threshold separates quo from a FAR page, so it would trade one missed collapse for routine false warnings. The numbers are recorded next to _COLLAPSE_RUN_LEN so it is not re-proposed. Gate: ruff, mypy, 275 tests, consistency all green. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 57 ++++++++++++++- README.md | 10 ++- src/tearsheet/content.py | 135 +++++++++++++++++++++++++++++++---- src/tearsheet/crawl.py | 36 +++++++++- src/tearsheet/scrape.py | 20 +++++- src/tearsheet/structured.py | 14 +++- tests/test_adversarial.py | 33 +++++++-- tests/test_crawl.py | 91 ++++++++++++++++++++++++ tests/test_quality.py | 136 ++++++++++++++++++++++++++++++++++++ tests/test_scrape.py | 99 ++++++++++++++++++++++++++ tests/test_structured.py | 40 +++++++++++ 11 files changed, 641 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6984fa..d71f7d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,13 +37,68 @@ failure. For the trust model and the evaluation harness behind those calls, see satisfies, so a 92% body collapse passed CI unnoticed. It now pins real body sentences, total guard silence, and that silence comes from the cluster floor rather than a toothless page. +### Added +- **`crawl` now runs the same guards as `scrape`.** It previously imported `extract_content` + alone — no `looks_blocked`, no `assess_extraction` — so a consent banner or bot wall was + written to disk as a numbered page file, cached, and listed in `INDEX.md` with an ordinary + token count. Walls are now refused: never written, never cached, their links never followed, + and reported as `refused: ()` with a `refused: N` count in the summary. Page + warnings ride in the index too. This mattered most on `crawl` precisely because its output is + read back off disk later rather than re-fetched — a wall written once is a wall trusted for weeks. +- **`extract` refuses bot walls** instead of caching them. It wrote to the same `pages.html` row + `scrape` reads, so one `extract` call could poison both verbs, and it returned the wall's own + structured data — usually `"no structured data found"`, which reads as *this page has none* + rather than *we never saw the page*. + +### Fixed +- **Money matching had four blind spots**, each of which silently disarmed the dropped-price + guard on the page class it exists for. Figures are now canonicalised (symbol-first, + space-free) via `money_figures()`, because `$ 49` and `$49` are the same price: + - `$49` — ordinary pricing-grid markup. Tag stripping inserts a + space and the pattern demanded a digit immediately after the symbol, so it matched **nothing**. + - `$49,` in prose kept the trailing comma, so the same figure compared unequal against its + table rendering and the guard reported a **false** drop. + - `19,99 €` — the German/French suffix form, on exactly the `tarif`/`preise` pages the + title-arming path targets — never matched at all. + - `$1.1 billion` matched as `$1`, silently changing the figure. +- **`tarif` matched English "tariff"/"tariffs"** (no word boundary), handing news articles the + low arming floor the design explicitly reserves for pricing pages — and tariff coverage is + precisely the genre that scatters real dollar figures through prose. +- **Guard thresholds are now link-invariant.** `_STUB_MAX_MARKDOWN`, `_BLOCK_MAX_CHARS` and + `_CONSENT_MAX_CHARS` are absolute character counts while `scrape` passes the caller's + `include_links` straight through; link syntax inflates markdown by ~70%, so the same failing + page went silent purely because the caller asked for links. +- **`html_to_text` is charset-aware** (declared charset, then cp1252) instead of utf-8-only. + This was pinned as a cosmetic `raw` limitation, which understated it: this function produces + the `page_text` that ARMS the price, stub and gated guards, so a latin-1 pricing page had its + `£` figures replaced before counting and **no guard could arm**. +- **`fresh=True` now evicts a poisoned row.** `replace_poisoned` was only reachable through the + cache-read branch that `fresh=True` skips, so the caller got correct content while the cache + kept the wall for the rest of the 7-day TTL and the next ordinary scrape replayed it. +- The raw-body wall check now covers every content type the extractor accepts + (`application/xhtml+xml`, empty) rather than `text/*` alone — defense in depth; the + post-extraction backstop already caught the reachable cases. + +Calibration: replaying all 194 cached pages produced **zero guard-verdict changes**, so none of +the above widened or narrowed what fires on the real corpus. Corpus-wide distinct figures went +363 → 377, every delta verified legitimate (recovered split-markup figures, `$1.1` no longer +truncated to `$1`, and trailing-comma duplicates correctly merging). + ### Known issues - **The collapsed-column guard has no real-page coverage under 2.2.0.** Quo is just as lossy (still 4 of 24 figures, 1 of 3 plan names) but emits the repeated cell once instead of three times, so the repetition signature the guard keys on is gone — on quo and on every other page in the cache replay. The dropped-price guard still catches quo, so the page is not silent. The quo pin was narrowed deliberately and the substantive loss pinned separately rather than - the assertion being deleted quietly. + the assertion being deleted quietly. A replacement signal — a short cell value repeated inside + a small window of visible text but present fewer times in the markdown, i.e. detecting the + deduplication itself — was built and **measured and rejected**: it separates the fixtures + cleanly (quo 19, heyrosie 1, article_peripheral 0) and then fires just as hard on ordinary + federal prose (`acquisition.gov/far/52.227-20` 19, `artificialintelligenceact.eu` 16, NASA + SBIR 13). No threshold separates quo from a FAR page. Shipping it would trade one missed + collapse for routine false warnings — the trade that destroys the value of every other + warning. Same reasoning that rejected the bare yield ratio in v0.1.2; the numbers are recorded + next to `_COLLAPSE_RUN_LEN` so it is not re-proposed. ## [0.1.5] — 2026-07-24 diff --git a/README.md b/README.md index 1d5b8b8..d59c31c 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ no telemetry. > *tearsheet (n.): a page torn from a publication and filed as proof it ran.* -**Trust status:** qualified for heavy usage 2026-07-16 — 252 tests, a falsifiable live +**Trust status:** qualified for heavy usage 2026-07-16 — 275 tests, a falsifiable live eval harness (verdict GREEN), and zero fabrications across the tool's entire recorded history. Its documented failure mode is *omission*, and the guards exist to make every omission loud. See [Trust](#trust). @@ -129,6 +129,10 @@ a research tool for reading the public web — not for evading paywalls or bot d reported and never cached, and previously poisoned cache rows are evicted, not replayed. Note `--raw` deliberately uses the plain fetch, not the browser: a rendered DOM can be *worse* (on smith.ai the consent overlay replaced the pricing table the raw fetch still carried). + Since 2026-08-11 the money matcher also handles currency split from its digits by markup + (`$49`), the `19,99 €` suffix form, and non-UTF-8 pages — each of + which previously left the guard unable to arm at all — and `crawl` and `extract` run the same + wall guards as `scrape` rather than saving a banner to disk as a page. - **Listings and rosters can come back as a boilerplate stub.** A page with no prices, no table, and no wall gave the earlier guards nothing to arm on, so a directory page could return its marketing sentence and silently drop every row. Measured 2026-07-24 across the @@ -165,7 +169,7 @@ a research tool for reading the public web — not for evading paywalls or bot d "Can it be trusted for heavy usage?" is a measurement here, not a feeling. -- **Offline suite (252 tests, runs in the gate)**: guard boundary pins, cache-poisoning +- **Offline suite (275 tests, runs in the gate)**: guard boundary pins, cache-poisoning regressions, truncation honesty, charset torture, structure torture, adversarial robustness — enforced fully offline by a loopback-only socket guard. The five REAL pages that defined the tool's probation (quo, smith.ai, dialpad, heyrosie, a LinkedIn @@ -199,7 +203,7 @@ for figures you'll quote, treat a suspiciously small extraction of a rich page a .venv/bin/ruff check src tests && .venv/bin/mypy && .venv/bin/python -m pytest ``` -TDD throughout; the default suite (252 tests) runs entirely offline — `httpx.MockTransport`, +TDD throughout; the default suite (275 tests) runs entirely offline — `httpx.MockTransport`, fixture HTML, and a conftest socket guard that fails any test reaching for a non-loopback address. Extras: `pytest -m playwright` (real chromium, local server), `pytest -m live` (real network). The live trust evaluation lives in `evals/` (see [Trust](#trust)). diff --git a/src/tearsheet/content.py b/src/tearsheet/content.py index 595bae8..a32b574 100644 --- a/src/tearsheet/content.py +++ b/src/tearsheet/content.py @@ -8,7 +8,30 @@ _SCRIPT_STYLE = re.compile(r"<(script|style)\b.*?", re.S | re.I) _TAG = re.compile(r"<[^>]+>") _WHITESPACE = re.compile(r"\s+") -_MONEY = re.compile(r"[$€£][0-9][0-9,]*(?:\.[0-9]{2})?") +# Money matching. Four blind spots were measured out of this on 2026-08-11; each one +# defeated the dropped-price guard SILENTLY on the page class it exists for: +# +# 1. `$49` — ordinary pricing-grid markup. Tag stripping puts +# a space between symbol and digits, and the old pattern demanded a digit +# immediately after the symbol, so it matched NOTHING on such a page. Hence `\s?`. +# 2. `$49,` in prose — `[0-9][0-9,]*` swallowed the trailing comma, so the same figure +# compared unequal against its table rendering and the guard reported a FALSE drop. +# The amount now has to end in a digit. +# 3. `19,99 €` — the standard German/French suffix form, on pages _PRICING_TITLE +# deliberately targets (`tarif`, `preise`). Never matched at all. Hence the second +# alternative. +# 4. Comparison requires a canonical form: `$ 49` on the page and `$49` in the markdown +# are the same price. Use money_figures()/_money_hits(), never _MONEY.findall — +# the raw match text is not comparable across renderings. +# +# The amount body allows `.` and `,` internally but must start AND end with a digit, which +# covers `49`, `1,200`, `10,500.00`, `19,99` and `1.234,56` without eating punctuation. +_AMOUNT = r"[0-9](?:[0-9.,]*[0-9])?" +_MONEY = re.compile( + rf"(?P[$€£])\s?(?P{_AMOUNT})|(?P{_AMOUNT})\s?(?P[€£])" +) +# Markdown link syntax, stripped before any absolute character threshold is applied. +_MD_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)") # Strong phrases + a size guard, mirroring fetch.looks_blocked: a page that merely # DISCUSSES cookie consent is long; a page that IS a consent wall is short. @@ -65,7 +88,12 @@ # window — because plan cards diluted by marketing prose can spread real figures more # than 1,500 chars apart. Articles never get this path (the LinkedIn FP protection): # their titles don't say "pricing". -_PRICING_TITLE = re.compile(r"pricing|plans? and pricing|tarif|preise", re.I) +# `tarif` carried no word boundary, so English "tariff"/"tariffs" matched it and handed +# NEWS ARTICLES the low floor the comment above promises they never get — and tariff +# coverage is exactly the genre that scatters real dollar figures through prose. `tarifs?\b` +# keeps the French singular/plural and rejects the double-f English word. (`plans? and +# pricing` was also dropped: anything matching it already matched the leading `pricing`.) +_PRICING_TITLE = re.compile(r"pricing|tarifs?\b|tarification|preise", re.I) _MONEY_MIN_TITLED = 3 # Stub-extraction guard (the roster/directory class, 2026-07-24 audit). A content-bearing @@ -114,6 +142,30 @@ # A collapsed table column reads as a run of identical consecutive rows # (quo: `Unlimited* / Unlimited* / Unlimited*`). One run happens naturally; two is a pattern. +# +# ⚠ KNOWN GAP since trafilatura 2.2.0 (2026-08-11): this guard has NO real-page coverage. +# 2.2.0 deduplicates identical consecutive blocks, so quo — the page this was built on — +# now emits that cell ONCE. The page is just as lossy (still 4 of 24 figures, 1 of 3 plan +# names); only the signature is gone. A full-cache replay confirms the guard fires on no +# page it used to catch. The dropped-price guard still catches quo, so it is not silent. +# +# REJECTED REPLACEMENT — measured 2026-08-11, do not re-propose without new evidence: +# "a short cell value repeated >=3x within a 300-char window of visible text but present +# fewer times in the markdown" (i.e. detect the dedup itself). It separates the FIXTURES +# beautifully — quo 19 hits, heyrosie 1, article_peripheral 0 — and then falls apart on +# the real corpus, because ordinary prose pages repeat short strings in nav and headings: +# +# quo.com/pricing 19 <- the page we want +# vmware.com/.../sovereign-cloud 22 +# acquisition.gov/far/52.227-20 19 <- a regulation page, not a table +# artificialintelligenceact.eu/art/99 16 +# nasa.gov/sbir_sttr/phase-i 13 +# +# No threshold separates quo from the FAR pages, and at >=10 it would fire on 20 of 194 +# pages — mostly the federal prose corpus this tool is actually used for. Shipping it +# would trade a missed collapse for routine false warnings, which is the trade that +# destroys the value of every OTHER warning. Same reasoning that rejected the bare yield +# ratio in v0.1.2. (It also costs ~63ms/page, versus microseconds for the run scan.) _COLLAPSE_RUN_LEN = 3 _COLLAPSE_MIN_RUNS = 2 _COLLAPSE_MAX_LINE = 80 @@ -135,6 +187,37 @@ class ExtractionQuality: warnings: list[str] = field(default_factory=list) +_META_CHARSET = re.compile(rb"""]+charset=["']?\s*([a-zA-Z0-9_-]+)""", re.I) + + +def _decode(html: bytes) -> str: + """Decode page bytes, honouring a declared charset. + + This used to be a bare `decode("utf-8", errors="replace")`, which quietly diverged + from extract_content — that hands trafilatura raw BYTES precisely so its charset + detection runs. The two sides disagreeing is not just a `raw` cosmetic issue: this + function produces the `page_text` that ARMS the price, stub and gated guards, so a + latin-1 pricing page had every `£` replaced before counting, `on_page` came back + empty, and no guard could arm on a page whose figures were all right there. + + utf-8 first because almost every page is utf-8 and this runs on every scrape; the + detection path costs nothing until a page actually fails to decode. + """ + try: + return html.decode("utf-8") + except UnicodeDecodeError: + pass + declared = _META_CHARSET.search(html[:4096]) + if declared: + try: + return html.decode(declared.group(1).decode("ascii", "ignore"), errors="replace") + except LookupError: + pass + # cp1252 over latin-1: real-world legacy Western pages use the Windows superset + # (curly quotes, en dashes) and it decodes the latin-1 range identically. + return html.decode("cp1252", errors="replace") + + def html_to_text(html: bytes) -> str: """Visible text of a page: scripts/styles gone, tags dropped, entities decoded. @@ -143,14 +226,34 @@ def html_to_text(html: bytes) -> str: """ if not html: return "" - text = _SCRIPT_STYLE.sub(" ", html.decode("utf-8", errors="replace")) + text = _SCRIPT_STYLE.sub(" ", _decode(html)) text = html_module.unescape(_TAG.sub(" ", text)) return _WHITESPACE.sub(" ", text).strip() +def _money_hits(text: str) -> list[tuple[int, str]]: + """(position, canonical figure) for every money amount, symbol-first and space-free. + + Canonicalising here is what makes page-vs-extraction comparison meaningful: `$ 49` + from split markup, `$49` from prose and `49 €` from a European grid all have to + reduce to one comparable token. + """ + hits = [] + for match in _MONEY.finditer(text): + symbol = match.group("sym") or match.group("sym2") + amount = match.group("amt") if match.group("amt") is not None else match.group("amt2") + hits.append((match.start(), f"{symbol}{amount}")) + return hits + + +def money_figures(text: str) -> set[str]: + """The distinct canonical money figures in a string.""" + return {figure for _, figure in _money_hits(text)} + + def _has_price_cluster(page_text: str) -> bool: """True when some 1,500-char window of visible text holds >= 4 distinct figures.""" - hits = [(m.start(), m.group()) for m in _MONEY.finditer(page_text)] + hits = _money_hits(page_text) if len(hits) < _MONEY_MIN_CLUSTERED: return False left = 0 @@ -195,13 +298,19 @@ def assess_extraction(html: bytes, extracted: ExtractedContent | None) -> Extrac return quality markdown = extracted.markdown page_text = html_to_text(html) - lowered = markdown.lower() - - if len(markdown) < _CONSENT_MAX_CHARS and any(p in lowered for p in _CONSENT_PHRASES): + # Every absolute character threshold below is measured against link-stripped markdown. + # scrape passes the caller's include_links straight through, and link syntax inflates + # markdown by ~70% — enough for a stub to clear _STUB_MAX_MARKDOWN or a banner to clear + # _CONSENT_MAX_CHARS purely because the caller asked for links. The thresholds are + # calibrated on prose length, so they have to see prose length. + measured = _MD_LINK.sub(r"\1", markdown) + lowered = measured.lower() + + if len(measured) < _CONSENT_MAX_CHARS and any(p in lowered for p in _CONSENT_PHRASES): quality.consent_wall = True return quality - if len(markdown) < _BLOCK_MAX_CHARS and any(p in lowered for p in _BLOCK_PHRASES): + if len(measured) < _BLOCK_MAX_CHARS and any(p in lowered for p in _BLOCK_PHRASES): quality.block_wall = True return quality @@ -214,23 +323,23 @@ def assess_extraction(html: bytes, extracted: ExtractedContent | None) -> Extrac if ( len(page_text) >= _STUB_MIN_PAGE_TEXT - and len(markdown) < _STUB_MAX_MARKDOWN - and len(markdown) / len(page_text) < _STUB_MAX_RATIO + and len(measured) < _STUB_MAX_MARKDOWN + and len(measured) / len(page_text) < _STUB_MAX_RATIO ): quality.warnings.append( - f"extraction is a {len(markdown)}-char stub of a page holding ~{len(page_text)} " + f"extraction is a {len(measured)}-char stub of a page holding ~{len(page_text)} " "chars of visible text — names, rows, or listings on this page are probably " "missing entirely. Use raw=true, or fetch independently." ) - on_page = set(_MONEY.findall(page_text)) + on_page = money_figures(page_text) title_armed = ( extracted.title is not None and _PRICING_TITLE.search(extracted.title) is not None and len(on_page) >= _MONEY_MIN_TITLED ) if title_armed or _has_price_cluster(page_text): - kept = on_page & set(_MONEY.findall(markdown)) + kept = on_page & money_figures(markdown) if len(kept) / len(on_page) < _MONEY_MIN_RETAINED: quality.warnings.append( f"extraction dropped {len(on_page) - len(kept)} of {len(on_page)} distinct prices " diff --git a/src/tearsheet/crawl.py b/src/tearsheet/crawl.py index 736c4b0..33b72fb 100644 --- a/src/tearsheet/crawl.py +++ b/src/tearsheet/crawl.py @@ -14,8 +14,8 @@ from tearsheet.cache import Cache, CachedPage from tearsheet.config import Settings, get_settings -from tearsheet.content import extract_content -from tearsheet.fetch import fetch_url, needs_render +from tearsheet.content import assess_extraction, extract_content +from tearsheet.fetch import fetch_url, looks_blocked, needs_render from tearsheet.mapper import extract_links, host_allowed from tearsheet.output import estimate_tokens, slugify from tearsheet.robots import get_policy @@ -31,12 +31,15 @@ class _PageRecord: title: str path: str url: str + warnings: list[str] = field(default_factory=list) @dataclass class _State: pages: list[_PageRecord] = field(default_factory=list) errors: list[tuple[str, str]] = field(default_factory=list) + # (path, reason) for pages refused as consent/bot walls — see _guard_verdict. + refused: list[tuple[str, str]] = field(default_factory=list) skipped: int = 0 in_flight: int = 0 visited: set[str] = field(default_factory=set) @@ -139,11 +142,31 @@ async def process(current_url: str, depth: int) -> None: state.skipped += 1 return body = result.body or b"" + # Guard parity with scrape (2026-08-11 audit): crawl used to run extraction + # alone, so a consent banner or bot wall was written to disk as a page, + # cached, and indexed with an ordinary token count. A wall is never the page + # — refuse it, keep it out of the cache, and say so in INDEX.md. This matters + # more here than in scrape: crawl output is read back off disk later instead + # of being re-fetched, so a wall written now is a wall trusted for weeks. + if looks_blocked(body): + async with lock: + state.refused.append((display_path, "blocked by bot protection")) + return extracted = extract_content(body, url=result.final_url) if extracted is None or needs_render(body, extracted.markdown): async with lock: state.skipped += 1 return + quality = assess_extraction(body, extracted) + if quality.consent_wall or quality.block_wall: + reason = ( + "consent/cookie wall" + if quality.consent_wall + else "blocked by bot protection (post-extraction backstop)" + ) + async with lock: + state.refused.append((display_path, reason)) + return # not written, not cached, and its links are wall links title = extracted.title or display_path tokens = estimate_tokens(extracted.markdown) @@ -155,6 +178,7 @@ async def process(current_url: str, depth: int) -> None: title=title, path=display_path, url=result.final_url, + warnings=list(quality.warnings), ) state.pages.append(record) front_matter = ( @@ -212,11 +236,19 @@ async def worker() -> None: ] summary = ( f"crawl: {root_host} pages: {len(state.pages)} errors: {len(state.errors)}" + f" refused: {len(state.refused)}" f" skipped(robots/dupe/type): {state.skipped}" ) lines = [summary, f"dir: {out_dir}", *index_lines] for path, reason in state.errors: lines.append(f"errors: {path} ({reason})") + # Refusals and warnings ride in the index because the index IS what gets read — + # a page missing from a crawl with no explanation reads as "the site didn't have it". + for path, reason in state.refused: + lines.append(f"refused: {path} ({reason}) — not saved, not cached") + for record in state.pages: + for warning in record.warnings: + lines.append(f"warning: {record.path} — {warning}") report = "\n".join(lines) (out_dir / "INDEX.md").write_text(report + "\n") diff --git a/src/tearsheet/scrape.py b/src/tearsheet/scrape.py index 482b293..23cc082 100644 --- a/src/tearsheet/scrape.py +++ b/src/tearsheet/scrape.py @@ -119,7 +119,14 @@ async def scrape( body = result.body or b"" ct = result.content_type - if ct.startswith("text/") and looks_blocked(body): + # Check every type the extractor will go on to accept, not just text/*: the + # branch below also lets `application/xhtml+xml` and an empty content-type + # through. The post-extraction backstop does catch these walls today (they + # extract short), so this is defense in depth rather than a plugged leak — + # it closes the case of a wall that extracts past _BLOCK_MAX_CHARS. + if (not ct or ct.startswith("text/") or ct == "application/xhtml+xml") and looks_blocked( + body + ): return ( f"blocked by bot protection (final url: {result.final_url});" " the site may offer an official API — try that or a manual fetch" @@ -140,7 +147,8 @@ async def scrape( html=None, markdown=text, title=None, - ) + ), + force=replace_poisoned or fresh, ) return _format( url=result.final_url, @@ -227,7 +235,13 @@ async def scrape( markdown=extracted.markdown, title=extracted.title, ), - force=replace_poisoned, + # `fresh` forces too: replace_poisoned is only reachable through the cache-read + # branch above, which fresh=True skips entirely. Without this, a fresh re-fetch + # returned correct content to the caller while cache.py's playwright-beats-httpx + # preference silently kept a poisoned row — so the NEXT ordinary scrape replayed + # the wall for the rest of the TTL. "Bypass the cache and refetch" has to mean + # the refetch wins. + force=replace_poisoned or fresh, ) warnings = list(quality.warnings) if warning: diff --git a/src/tearsheet/structured.py b/src/tearsheet/structured.py index 05ff5cc..f9083c7 100644 --- a/src/tearsheet/structured.py +++ b/src/tearsheet/structured.py @@ -13,7 +13,7 @@ from tearsheet.cache import Cache, CachedPage from tearsheet.config import Settings, get_settings -from tearsheet.fetch import fetch_url +from tearsheet.fetch import fetch_url, looks_blocked from tearsheet.render import RenderUnavailableError, render_page DEFAULT_TYPES = ["json-ld", "opengraph", "microdata", "tables"] @@ -49,6 +49,18 @@ async def extract_page( if result.status >= 400: return json.dumps({"url": url, "error": f"HTTP {result.status}"}) body = result.body or b"" + # A wall is not the page — and extract writes to the SAME pages.html row scrape + # reads, so caching one here poisons both verbs. Previously this returned the + # wall's own tables/JSON-LD as structured data (usually "no structured data + # found", which reads as "this page has none" rather than "we never saw it"). + if looks_blocked(body): + return json.dumps( + { + "url": result.final_url, + "error": "blocked by bot protection; the site may offer an official " + "API — try that or a manual fetch", + } + ) cache.put_page( CachedPage( url=url, diff --git a/tests/test_adversarial.py b/tests/test_adversarial.py index ca63b36..baf38de 100644 --- a/tests/test_adversarial.py +++ b/tests/test_adversarial.py @@ -190,15 +190,34 @@ def test_utf8_bom_is_clean(self) -> None: assert "BOM page content" in extracted.markdown assert "" not in extracted.markdown - def test_raw_path_is_utf8_only_pinned_limitation(self) -> None: - """PINNED LIMITATION (trust-suite review 2026-07-16): html_to_text — the raw - escape hatch — decodes utf-8-with-replace only. Non-UTF8 pages come back as - mojibake through --raw (the extractor path handles them; raw does not). - When this pin fails, charset detection was added to the raw path: update - the README known-issues entry and delete this test.""" + def test_non_utf8_pages_decode_correctly(self) -> None: + """WAS a pinned limitation ("raw is utf-8 only, the extractor path handles them; + raw does not"), retired 2026-08-11. That framing understated it: html_to_text + also produces the `page_text` that ARMS the price/stub/gated guards, so a + latin-1 pricing page had its `£` figures replaced before counting and NO guard + could arm on it. Now charset-aware — declared charset first, cp1252 fallback.""" html = ("

café £3

").encode("latin-1") text = html_to_text(html) - assert "café" not in text # mojibake today, documented + assert "café £3" in text + + def test_declared_charset_is_honoured(self) -> None: + html = ( + "

τιμή

" + ).encode("iso-8859-7") + assert "τιμή" in html_to_text(html) + + def test_latin1_pricing_page_can_still_arm_the_guard(self) -> None: + """The reason this matters: the guard reads html_to_text, not the extraction.""" + from tearsheet.content import ExtractedContent, assess_extraction + + body = "Plans: £49 then £99 then £149 and finally £199 per month." + html = f"Pricing

{body}

".encode( + "latin-1" + ) + quality = assess_extraction( + html, ExtractedContent(markdown="Plans", title="Pricing", description=None) + ) + assert any("price" in w for w in quality.warnings) class TestStructureTorture: diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 0116042..20a5e1d 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -177,3 +177,94 @@ async def test_custom_output_dir(self, site: httpx.MockTransport, tmp_path: Path out = await run_crawl(site, max_pages=3, output_dir=str(target)) assert crawl_dir_of(out) == target assert list(target.glob("[0-9]*.md")) + + +# ── Guard parity with scrape (2026-08-11 audit) ────────────────────────────── +# +# crawl imported only extract_content: no looks_blocked, no assess_extraction. A consent +# banner or bot wall was written to disk as a normal page file, cached, and listed in +# INDEX.md with an ordinary token count — the exact "pass the wall off as the page" +# failure the scrape guards exist to prevent, on the verb that writes to disk and is read +# back later instead of re-scraped. + +CONSENT_WALL = ( + b"Cookies

" + b"We use cookies to analyze site traffic and personalize content. " + b"Accept all cookies to continue, or manage preferences to choose which " + b"categories you allow." + b"

" +) + +BOT_WALL = ( + b"Just a moment

" + b"Verify you are a human. Complete the CAPTCHA below to continue to the site." + b"

" +) + +WALLED_SITE: dict[str, bytes] = { + "/": page("Home", ["/docs/good", "/docs/consent", "/docs/bot"]), + "/docs/good": page("Good Page", []), + "/docs/consent": CONSENT_WALL, + "/docs/bot": BOT_WALL, +} + + +@pytest.fixture +def walled_site() -> httpx.MockTransport: + def handler(request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/robots.txt": + return httpx.Response(200, content=b"", headers={"content-type": "text/plain"}) + body = WALLED_SITE.get(path) + if body is None: + return httpx.Response(404, text="nope") + return httpx.Response(200, content=body, headers={"content-type": "text/html"}) + + return httpx.MockTransport(handler) + + +class TestCrawlRefusesWalls: + async def test_wall_pages_are_never_written_to_disk( + self, walled_site: httpx.MockTransport + ) -> None: + out = await run_crawl(walled_site, max_pages=10, max_depth=2) + d = crawl_dir_of(out) + written = "\n".join(p.read_text() for p in d.glob("[0-9]*.md")) + assert "We use cookies" not in written, "a consent banner was saved as a page" + assert "Complete the CAPTCHA" not in written, "a bot wall was saved as a page" + + async def test_only_the_real_page_counts(self, walled_site: httpx.MockTransport) -> None: + out = await run_crawl(walled_site, max_pages=10, max_depth=2) + d = crawl_dir_of(out) + # home + good page; the two walls must not inflate the page count + assert len(list(d.glob("[0-9]*.md"))) == 2 + assert "pages: 2" in out + + async def test_refusals_are_reported_not_silent( + self, walled_site: httpx.MockTransport + ) -> None: + out = await run_crawl(walled_site, max_pages=10, max_depth=2) + assert "refused: 2" in out + assert "/docs/consent" in out and "/docs/bot" in out + assert "consent" in out.lower() and "bot protection" in out.lower() + + async def test_index_json_excludes_walls(self, walled_site: httpx.MockTransport) -> None: + out = await run_crawl(walled_site, max_pages=10, max_depth=2) + index = json.loads((crawl_dir_of(out) / "index.json").read_text()) + assert all("consent" not in e["url"] and "bot" not in e["url"] for e in index) + + async def test_walls_are_never_cached(self, walled_site: httpx.MockTransport) -> None: + """A cached wall is worse than a refused one: scrape would replay it later.""" + from tearsheet.cache import Cache + + out = await run_crawl(walled_site, max_pages=10, max_depth=2) + assert "refused: 2" in out + cache = Cache(get_settings().cache_db) + try: + for url in ( + "https://docs.example.com/docs/consent", + "https://docs.example.com/docs/bot", + ): + assert cache.get_page(url, 86400) is None, f"wall cached: {url}" + finally: + cache.close() diff --git a/tests/test_quality.py b/tests/test_quality.py index 8736272..6542bf9 100644 --- a/tests/test_quality.py +++ b/tests/test_quality.py @@ -335,3 +335,139 @@ def test_tiny_page_cannot_arm_the_guard(self) -> None: body = "Hello. " * 20 # ~140 chars of visible text quality = assess_extraction(page(body), extracted("Hello.")) assert not any("stub" in w for w in quality.warnings) + + +# ── Money-matching blind spots found in the 2026-08-11 audit ───────────────── +# +# All four defeat the dropped-price guard on the exact page class it exists for, +# and every one of them fails SILENTLY — the guard cannot arm, so nothing warns. + +from tearsheet.content import ExtractedContent, money_figures # noqa: E402 + + +class TestMoneyMatching: + def test_currency_split_from_digits_by_markup(self) -> None: + """`$49` is ordinary pricing-grid markup. Tag + stripping inserts a space, so `$ 49` reached a regex demanding a digit + immediately after the symbol and matched nothing at all.""" + html = ( + b"
" + b"$49" + b"$99" + b"
" + ) + assert money_figures(html_to_text(html)) == {"$49", "$99"} + + def test_nbsp_between_symbol_and_digits(self) -> None: + html = b"

€ 49 and £ 99

" + assert money_figures(html_to_text(html)) == {"€49", "£99"} + + def test_trailing_comma_is_not_part_of_the_figure(self) -> None: + """`[0-9][0-9,]*` swallowed a following comma, so the SAME price compared + unequal against its table rendering and the guard reported a false drop.""" + prose = "Plans cost $49, $99, $149, $199 per month." + table = "| $49 | $99 | $149 | $199 |" + assert money_figures(prose) == money_figures(table) + + def test_thousands_separators_still_match(self) -> None: + assert money_figures("$1,200 and $10,500.00") == {"$1,200", "$10,500.00"} + + def test_suffix_currency_matches(self) -> None: + """`19,99 €` is the standard German/French form — and _PRICING_TITLE + deliberately targets `tarif`/`preise` pages, which is exactly where it appears.""" + assert money_figures("Ab 19,99 € pro Monat, dann 29,99 €") == {"€19,99", "€29,99"} + + def test_a_false_drop_is_not_reported_for_punctuation_alone(self) -> None: + page_text = "Plans cost $49, $99, $149, $199 per month across the lineup." + markdown = "| $49 | $99 | $149 | $199 |" + quality = assess_extraction( + page(page_text), ExtractedContent(markdown=markdown, title="Pricing", description=None) + ) + assert not any("price" in w for w in quality.warnings), ( + "every figure survived; a warning here is a false positive" + ) + + +class TestPricingTitleArming: + def test_tariff_article_does_not_get_the_low_floor(self) -> None: + """`tarif` had no word boundary, so English "tariffs" matched — handing news + articles the >=3-figures-page-wide arming path the design says they never get + ("Articles never get this path (the LinkedIn FP protection)"). Tariff coverage + is precisely the genre that scatters dollar figures through prose.""" + body = ( + "Trade groups said the measure covers $250 of components. " + + "Analysts discussed the policy at length. " * 40 + + "A second filing referenced $1,200 in duties. " + + "Ordinary reporting continues here for a while. " * 40 + + "A third mentioned $4,500 for the quarter." + ) + quality = assess_extraction( + page(body), + ExtractedContent( + markdown="Trade groups said the measure covers the components.", + title="Trump's new tariffs hit steel importers", + description=None, + ), + ) + assert not any("price" in w for w in quality.warnings) + + def test_real_french_pricing_title_still_arms(self) -> None: + body = "Nos tarifs: 19,99 € puis 29,99 € et enfin 49,99 € par mois." + quality = assess_extraction( + page(body), + ExtractedContent(markdown="Nos tarifs", title="Tarifs et abonnements", description=None), + ) + assert any("price" in w for w in quality.warnings) + + +class TestThresholdsAreLinkInvariant: + """_STUB_MAX_MARKDOWN / _BLOCK_MAX_CHARS / _CONSENT_MAX_CHARS are absolute character + counts, but scrape passes the caller's include_links straight into extraction. Link + syntax inflates markdown by ~70%, so the SAME failing page went silent purely because + the caller asked for links. The calibration comment names this hazard for measuring; + the thresholds have to be invariant to it too.""" + + def test_stub_still_warns_when_links_inflate_the_markdown(self) -> None: + visible = "Real roster content. " * 120 # ~2,500 chars of visible text + stub_plain = "Meet the passionate team members who make it happen." + stub_linked = " ".join( + f"[{w}](https://example.com/a/very/long/tracking/path/{i})" + for i, w in enumerate(stub_plain.split()) + ) + assert len(stub_linked) > 500 # would clear the raw floor on length alone + quality = assess_extraction(page(visible), extracted(stub_linked)) + assert any("stub" in w for w in quality.warnings) + + def test_consent_wall_still_caught_when_links_inflate_it(self) -> None: + banner = CONSENT_BANNER + " " + " ".join( + f"[cookie policy {i}](https://example.com/legal/cookies/section/{i})" + for i in range(40) + ) + assert len(banner) > 2_000 + quality = assess_extraction(page(CONSENT_BANNER), extracted(banner)) + assert quality.consent_wall + + +class TestCollapsedColumnsKnownGap: + def test_the_run_signature_still_works_when_present(self) -> None: + """The guard is not broken — its input class is. Pinned so that if a future + extractor stops deduplicating, the guard is known to still catch it.""" + markdown = ( + "Calling\n\nUnlimited*\n\nUnlimited*\n\nUnlimited*\n\n" + "Messaging\n\nUnlimited*\n\nUnlimited*\n\nUnlimited*\n" + ) + quality = assess_extraction(page("Calling Unlimited"), extracted(markdown)) + assert any("column" in w for w in quality.warnings) + + def test_deduplicated_collapse_is_not_detected_documented(self) -> None: + """DOCUMENTED GAP (2026-08-11): when the extractor emits the repeated cell once, + nothing here can see it. The replacement signal was measured and rejected — it + fired on FAR and NASA prose pages as hard as on quo. See the comment above + _COLLAPSE_RUN_LEN. Quo stays covered by the dropped-price guard. + + If this test ever fails, a working collapse signal was found: delete it and the + rejection note. + """ + markdown = "Calling\n\nUnlimited*\n\nMessaging\n\nUnlimited*\n\nRecording\n\nManual\n" + quality = assess_extraction(page("Calling Unlimited"), extracted(markdown)) + assert not any("column" in w for w in quality.warnings) diff --git a/tests/test_scrape.py b/tests/test_scrape.py index e39a4ec..6b6b485 100644 --- a/tests/test_scrape.py +++ b/tests/test_scrape.py @@ -438,3 +438,102 @@ def handler(request: httpx.Request) -> httpx.Response: assert calls["count"] == 1 assert "cache" in out2 assert "linden trees bloom" in out2 + + +# ── Cache-correctness gaps found in the 2026-08-11 audit ───────────────────── + +CONSENT_BODY = ( + b"Cookies

" + b"We use cookies to analyze site traffic. Accept all cookies to continue, " + b"or manage preferences to choose categories." + b"

" +) + +GOOD_BODY = ( + b"Real Page
" + b"

Real Page

This is the genuine article body, long enough for the " + b"extractor to treat it as real content rather than boilerplate, with several " + b"sentences of ordinary prose carrying actual information for the reader.

" + b"
" +) + + +class TestFreshEvictsPoisonedRows: + """`fresh=True` returned correct content but could not overwrite a poisoned + playwright row: `replace_poisoned` was only ever set inside the `if not fresh:` + cache-read branch, so put_page got force=False and cache.py's playwright-beats-httpx + preference silently kept the poison for the rest of the 7-day TTL. The caller saw + good content; the NEXT ordinary scrape replayed the wall. + """ + + async def test_fresh_overwrites_a_playwright_wall_row(self) -> None: + import time + + from tearsheet.cache import Cache, CachedPage + from tearsheet.config import get_settings + + url = "https://example.com/poisoned" + cache = Cache(get_settings().cache_db) + try: + cache.put_page( + CachedPage( + url=url, + final_url=url, + fetched_at=int(time.time()), + status=200, + content_type="text/html", + via="playwright", # outranks httpx unless force=True + html=CONSENT_BODY, + markdown="We use cookies", + title="Cookies", + ) + ) + finally: + cache.close() + + transport = httpx.MockTransport( + lambda r: httpx.Response( + 200, content=GOOD_BODY, headers={"content-type": "text/html"} + ) + ) + out = await scrape(url, fresh=True, render="never", transport=transport) + assert "genuine article body" in out + + cache = Cache(get_settings().cache_db) + try: + row = cache.get_page(url, 86400) + assert row is not None + assert b"We use cookies" not in (row.html or b""), ( + "fresh=True left the poisoned playwright row in the cache" + ) + finally: + cache.close() + + +class TestWallDetectionCoversAllServedTypes: + """looks_blocked was gated on `ct.startswith("text/")` while the extractor path also + accepts `application/xhtml+xml` and an empty content-type, so the RAW-BODY check was + skipped for those. + + Honest scope note: these two cases were already reported correctly before that gate + was widened — the post-extraction block_wall backstop catches them because a wall + extracts to well under _BLOCK_MAX_CHARS. So this is a defense-in-depth pin, not a + regression test for a reachable silent failure. It guards the case the backstop + cannot see: a wall that extracts LONG. + """ + + BOT_BODY = ( + b"Attention

" + b"Verify you are a human. Complete the CAPTCHA to continue." + b"

" + ) + + @pytest.mark.parametrize("content_type", ["application/xhtml+xml", ""]) + async def test_wall_reported_for_non_text_html_types(self, content_type: str) -> None: + transport = httpx.MockTransport( + lambda r: httpx.Response( + 200, content=self.BOT_BODY, headers={"content-type": content_type} + ) + ) + out = await scrape("https://example.com/walled", render="never", transport=transport) + assert "blocked by bot protection" in out diff --git a/tests/test_structured.py b/tests/test_structured.py index 8e0459f..c6f24b4 100644 --- a/tests/test_structured.py +++ b/tests/test_structured.py @@ -138,3 +138,43 @@ def test_types_filter(self, fixture_bytes: Callable[[str], bytes]) -> None: def test_nothing_found_message(self) -> None: out = json.loads(extract_structured(b"

plain

", "https://x.com")) assert out.get("note") == "no structured data found" + + +# ── Wall poisoning via extract (2026-08-11 audit) ──────────────────────────── + +BOT_WALL_HTML = ( + b"Attention Required

" + b"Verify you are a human. Complete the CAPTCHA below to continue to the site." + b"

" +) + + +class TestExtractDoesNotPoisonTheCache: + """extract_page cached whatever it fetched with no wall check at all, so a bot wall + became the shared `pages.html` row that scrape and extract both read. scrape + self-heals on its next non-fresh read, but until then the wall is the cached page — + and extract itself happily returned the wall's tables and JSON-LD as structured data. + """ + + async def test_a_bot_wall_is_reported_not_cached( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + monkeypatch.setenv("TEARSHEET_HOME", str(tmp_path / "home")) + from tearsheet.cache import Cache + from tearsheet.config import get_settings + from tearsheet.structured import extract_page + + url = "https://example.com/walled" + transport = httpx.MockTransport( + lambda r: httpx.Response( + 200, content=BOT_WALL_HTML, headers={"content-type": "text/html"} + ) + ) + out = await extract_page(url, render="never", transport=transport) + assert "bot protection" in out.lower() or "blocked" in out.lower() + + cache = Cache(get_settings().cache_db) + try: + assert cache.get_page(url, 86400) is None, "extract cached a bot wall" + finally: + cache.close()