From 2e03a1b3cab7238ac1fe383763206e5e79e14a65 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:18:10 +0000 Subject: [PATCH 1/2] Stop counting a repeated transfers page as served; read the edge's 502 as a failed page On 2026-10-15 the default latest-transfers view (filtered by the site to players valued EUR 500,000+) held 20 transfers on one page with no pager, and ?page=2 served page 1 again. A page number past the end of the unfiltered listing is answered the same way, with the last page again. The run counted the repeat as a completed page and ended "complete". A page whose rows were all seen on earlier pages is now never counted. If the page before it had no next link the listing ended there (stop_reason listing_ended, exit 0, repeated_page / repeat_of_page / listing_pages in the sidecar); if it had one, a page was promised and not served (stop_reason page_repeated, exit 6). Same rule in all four entry points and in Playwright's --concurrency path, which also stops dispatching on a repeat. The edge's bare nginx 5xx page is recognised from its body. After the AWS WAF challenge clears, its status is gone, and a 502 left in the browser read as an empty page 1 while the run went on to report complete. The canary now also runs --mode transfers on the unfiltered listing and requires more than one page served, unique transfer ids and filled core fields. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/canary.yml | 80 ++++++++++ CHANGELOG.md | 27 ++++ README.md | 42 ++++- TROUBLESHOOTING.md | 22 ++- output_writer.py | 2 +- page_flow.py | 102 +++++++++++- playwright_scraper.py | 57 ++++++- product_parser.py | 34 ++++ puppeteer_scraper.py | 26 ++- scraper_api_client.py | 20 ++- selenium_scraper.py | 26 ++- smoke_test.py | 296 +++++++++++++++++++++++++++++++++++ 12 files changed, 703 insertions(+), 31 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index d8ba59e..ab3bb60 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -102,6 +102,26 @@ jobs: --out canary_run --dump-html canary_run.html echo "exit_code=$?" >> "$GITHUB_OUTPUT" + - name: Scrape the latest-transfers listing, unfiltered, 3 pages + id: transfers + # The other paginated mode, on its own pagination: ?page=N on a plain + # query string, and an end the site marks by serving the last page + # again. The default view is filtered by the site to players valued + # EUR 500,000+ and held one 20-row page on 2026-10-15, so it cannot + # show whether a second page is followed; the unfiltered view held + # 4 pages (77 transfers) the same day. + continue-on-error: true + env: + TRANSFERMARKT_CDP_ENDPOINT: ${{ secrets.TRANSFERMARKT_CDP_ENDPOINT }} + TRANSFERMARKT_PROXY: ${{ secrets.TRANSFERMARKT_PROXY }} + run: | + set +e + python3 playwright_scraper.py \ + --mode transfers --pages 3 \ + --url "https://www.transfermarkt.com/transfers/neuestetransfers/statistik?land_id=0&verein_land_id=0&wettbewerb_id=alle&minMarktwert=0&maxMarktwert=500000000" \ + --out canary_transfers --dump-html canary_transfers.html + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + - name: Interpret the exit code id: verdict env: @@ -213,6 +233,62 @@ jobs: sys.exit(1 if fail else 0) EOF + - name: Check the latest-transfers run + # Runs even when the market-values checks above failed, so one + # mode's break does not hide the other's. + if: ${{ !cancelled() }} + env: + CODE: ${{ steps.transfers.outputs.exit_code }} + run: | + if [ "$CODE" != "0" ]; then + echo "::error::--mode transfers exited $CODE (0 expected; 3 blocked, 4 zero rows, 5 nothing obtained, 6 partial — including stop_reason page_repeated, a page linked onward that served an earlier page again). canary_transfers.meta.json and canary_transfers_page*_debug.html in the artifacts say which." + exit 1 + fi + python3 - <<'EOF' + import json, sys + from pathlib import Path + + rows = json.loads(Path("canary_transfers.json").read_text()) + meta = json.loads(Path("canary_transfers.meta.json").read_text()) + fail = [] + + # Tests the pagination and the parser, not the transfer season: + # more than one page's worth of rows (25 a page) means a second + # page was followed, whatever the listing's depth that day. + if meta.get("pages_completed", 0) < 2: + fail.append(f"only {meta.get('pages_completed')} page(s) served " + f"(stop_reason={meta.get('stop_reason')!r}); the " + f"unfiltered listing had 4 on 2026-10-15") + if len(rows) <= 25: + fail.append(f"only {len(rows)} rows: no second page's worth") + if meta.get("status") != "complete" or meta.get("stop_reason") not in ( + "completed", "listing_ended"): + fail.append(f"status={meta.get('status')!r} " + f"stop_reason={meta.get('stop_reason')!r}") + skus = [r.get("sku") for r in rows] + if len(set(skus)) != len(skus): + fail.append("a transfer id appears twice: a page was counted twice") + pages = [r.get("page") for r in rows] + if pages != sorted(pages): + fail.append("rows are not in page order") + # Every row is built only when both ids are found, so these are + # 100%; the names come from the same cells. + for field, floor in (("sku", 1.0), ("player_id", 1.0), ("url", 1.0), + ("title", 0.95), ("from_club", 0.9), + ("fee_type", 1.0)): + got = sum(1 for r in rows if r.get(field) not in (None, "")) + if rows and got / len(rows) < floor: + fail.append(f"{field} present on only {got}/{len(rows)} rows " + f"(floor {floor:.0%})") + + for line in fail: + print(f"::error::{line}") + print(f"[canary transfers] {len(rows)} rows, " + f"status={meta.get('status')} stop_reason={meta.get('stop_reason')} " + f"pages={meta.get('pages_completed')}/3") + sys.exit(1 if fail else 0) + EOF + - name: Upload run artifacts (output, HTML snapshot, debug dumps) # Always, not just on failure: a run that returns the WRONG numbers # with no error is exactly the regression that needs the actual bytes @@ -227,5 +303,9 @@ jobs: canary_run.meta.json canary_run.html* canary_run_page*_debug.* + canary_transfers.json + canary_transfers.meta.json + canary_transfers.html* + canary_transfers_page*_debug.* if-no-files-found: ignore retention-days: 14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 667a073..961faa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,8 +37,35 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). > - **The canary fails when it is blocked.** A bare-runner exit 3 used to > end green with a notice. +> **Behaviour change (latest transfers):** a page whose rows were all seen on +> earlier pages is no longer counted. When the page before it had no next +> link the run ends `stop_reason: listing_ended` (exit 0, was +> `no_new_products`) with `repeated_page`, `repeat_of_page` and +> `listing_pages` in the sidecar; when the page before it linked onward the +> run ends `stop_reason: page_repeated`, exit 6 (was exit 0, "complete"). +> `pages_completed` no longer counts the repeated page. The edge's bare nginx +> 5xx page is now a failed page in every engine, never an empty one. + ### Fixed +- **`--mode transfers` counted a page the site did not serve.** Measured + 2026-10-15: the default latest-transfers view held 20 transfers (the site + filters it to players valued EUR 500,000+), had no pager, and answered + `?page=2` with page 1 again. The run fetched page 2, found nothing new, + and reported `pages_completed: 2`, `status: complete`. A page number past + the end of a longer listing is answered the same way, with the last page + again. A page that only repeats earlier rows is now recorded as the + listing's end (when the page before it had no next link) or as a page not + served (when it had one), in all four entry points and in Playwright's + `--concurrency` path, which also stops dispatching once a page repeats. + The README documents the filter and the unfiltered URL (4 pages, 77 + transfers that day). +- **A 502 after the AWS WAF challenge ended a run "complete" with page 1 + missing.** Measured 2026-10-15: the challenge cleared and the document left + in the browser was the edge's 155-byte nginx `502 Bad Gateway`. Its status + had rightly been discarded with the challenge's, so it classified as an + empty page, and the run carried on to pages 2-5 and reported 52 of 77 + rows as complete. The page is now recognised from its body. - **A failed write destroyed the previous good output.** `write_json`, `write_csv`, the `.meta.json` sidecar and `diff_runs.py --out` all opened their target with `open(path, "w")`, which truncates first: a dump that diff --git a/README.md b/README.md index 5e1e329..7923594 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,9 @@ python3 playwright_scraper.py --mode market-values # A club's full squad — id is transfermarkt's own numeric club id python3 playwright_scraper.py --mode club-squad --club-id 281 --season 2025 -# The latest-transfers ledger, 3 pages (safe to run with --concurrency) +# The latest-transfers ledger, up to 3 pages (safe to run with --concurrency). +# The default view is filtered to players valued EUR 500k+ and can fit on one +# page; see "How deep that listing goes" below for the unfiltered URL. python3 playwright_scraper.py --mode transfers --pages 3 --concurrency 3 # One player's full profile @@ -293,11 +295,35 @@ uses whichever path page 1 says to use; `--concurrency` remains usually unaddressable, but because running several workers concurrently against a mode that can rarely switch shape mid-run is untested here. -**`/statistik/neuestetransfers` was and remains simpler.** Its -`` is a bare `?page=2`, and page 2 fetched statelessly -shared **0** transfer ids with page 1 (25 rows each, measured 2026-09-10) — -genuinely, independently addressable on every capture taken of it so far. -That is the one mode `--concurrency` is enabled for. +**`/statistik/neuestetransfers` is simpler.** Its `` is a +bare `?page=2`, and page 2 fetched statelessly shared **0** transfer ids with +page 1 (25 rows each, measured 2026-09-10). That is the one mode +`--concurrency` is enabled for. + +**How deep that listing goes depends on the day, and on a filter you do not +see.** Measured 2026-10-15: + +- The default view applies the site's own market-value floor: its filter + form and tab links carry `minMarktwert=500000` (players valued at + EUR 500,000 or more). That day it held **20** transfers, under one 25-row + page, so it had no pager and no next link. That is the whole listing, not + a cap on what this scraper reads. +- Without the floor, the same listing had **4 pages, 77 transfers** (25, 25, + 25, 2), each page's next link a plain `&page=N` with the filters kept. + Pass that URL to read it: + `--mode transfers --url "https://www.transfermarkt.com/transfers/neuestetransfers/statistik?land_id=0&verein_land_id=0&wettbewerb_id=alle&minMarktwert=0&maxMarktwert=500000000" --pages 6`. +- A page number past the end is answered **HTTP 200 with the last page + again**: `?page=2` of the 20-row view served page 1, and `?page=5`, + `?page=401` and `?page=9999` of the unfiltered view all served page 4. + +So a page whose rows were all seen on earlier pages is never counted as a +fresh page. When the page before it had no next link, the listing ended +there: `stop_reason: listing_ended`, exit 0, and the sidecar names the +`repeated_page`, the page it repeated (`repeat_of_page`) and how many pages +the listing had (`listing_pages`). When the page before it DID link onward, +the site promised a page and served an old one: `stop_reason: +page_repeated`, exit 6. Either way `pages_completed` counts only the pages +the site actually served. --- @@ -578,13 +604,13 @@ push for the same four reasons. | Code | Meaning | |---|---| -| `0` | Complete | +| `0` | Complete (including a listing that ended before `--pages`: `stop_reason: listing_ended`) | | `1` | Crash | | `2` | Bad usage (including a URL that is not a supported Transfermarkt host) | | `3` | Blocked — a bot-check/challenge page | | `4` | Zero rows, and the site's answer really was nothing: an empty squad, or the club, player or URL asked for does not exist (the site's own 404 page, or a profile id answered with a redirect elsewhere; `stop_reason: not_found`) | | `5` | Nothing obtained. Either the content was never fetched or read — a load timeout, a dead exit, or a served page that shows rows the parser could not read (`stop_reason: parser_found_nothing`, a markup change) — or a 2Captcha product call failed on its own terms (the Fingerprint API, or a `--cdp-endpoint` connection: bad key, rate limit, a Scraping Browser profile already in use). Distinct from `3` and `4`: nothing can be concluded about the site from it. A captcha-SOLVE failure is deliberately NOT this — it's a warning and the run continues, reporting `3` only if the page really was blocked. | -| `6` | Partial — some pages fetched, then stopped early (including a later page that was refused, answered 404, or parsed to nothing while showing rows). Not necessarily a fault: for `--mode market-values`, this is the expected result on the rare run whose pagination link needs the fallback chain (see "the gallery pagination trap") and runs out of it; on the common run it does not happen at all. | +| `6` | Partial — some pages fetched, then stopped early (including a later page that was refused, answered 404, parsed to nothing while showing rows, or served an earlier page again although the page before it linked onward: `stop_reason: page_repeated`). Not necessarily a fault: for `--mode market-values`, this is the expected result on the rare run whose pagination link needs the fallback chain (see "the gallery pagination trap") and runs out of it; on the common run it does not happen at all. | **A run that finds nothing writes nothing.** Last known good output is not replaced with `[]`; `--allow-empty` is the opt-out. `.meta.json` diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index cc79fd9..10a4915 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -40,10 +40,24 @@ squad data yet answers 200 with a page that has no player rows on it, which is a correct "empty" answer, not a bug. **`--mode market-values` / `--mode transfers`** past page 1 — you may be one -page past the end of the ranking/listing. Both modes stop and report a -`no_new_products`/exhausted result rather than erroring, which is a -*complete* answer, not a truncated one — check `stop_reason` in the -`.meta.json` sidecar. +page past the end of the ranking/listing. The site answers a page number past +the end with the last page again (HTTP 200), so a page that brings no new +rows is not counted. If the page before it had no next link, that is the end +of the listing (`stop_reason: listing_ended`, exit 0, with `repeated_page` +and `listing_pages` in the `.meta.json` sidecar). An empty page ends the run +as `no_new_products`, also complete. + +**`--mode transfers` stops after one page of about 20 rows.** On 2026-10-15 +the default latest-transfers view (filtered by the site to players valued +EUR 500,000+) held 20 transfers: one page, no pager. That is the site's whole +listing for that filter. The unfiltered listing is longer; see "How deep that +listing goes" in the README for its URL. + +## Exit 6 with `stop_reason: page_repeated` + +A page linked to a next page, and the next page served rows already seen. +Pages were requested and not served, so the run is partial; the rows from +the pages that were served are kept. ## Exit 5 with `stop_reason: parser_found_nothing` diff --git a/output_writer.py b/output_writer.py index 58951f5..252758a 100644 --- a/output_writer.py +++ b/output_writer.py @@ -461,7 +461,7 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, # exist (its own 404 page, or a profile redirected away): a complete answer # with no rows, so exit 4, not a failed fetch. COMPLETE_STOP_REASONS = ("completed", "pagination_exhausted", "no_new_products", - "single_page_mode", "not_found") + "single_page_mode", "not_found", "listing_ended") def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, diff --git a/page_flow.py b/page_flow.py index 55ca41d..cccb41b 100644 --- a/page_flow.py +++ b/page_flow.py @@ -91,7 +91,10 @@ rel="next">` reads `?page=2`, a bare query parameter, and a stateless fetch of `?page=2` measured 0 shared transfer ids against page 1 (25 rows each) — genuinely, independently addressable on every capture taken of it so far, -which is the one mode `--concurrency` is enabled for. +which is the one mode `--concurrency` is enabled for. Its END is not a +missing page, though: a page number past the end serves the last page again +(measured 2026-10-15), which `PageSequence` below tells apart from a fresh +page. What this module deliberately does NOT contain ------------------------------------------------ @@ -380,6 +383,103 @@ def failure_reason(outcome) -> str: return f"blocked_{outcome.blocked_by}" +# --------------------------------------------------------------------------- +# A later page that serves an earlier page again +# --------------------------------------------------------------------------- +# The listing has ended: the page before the repeat carried no next-page +# link, and the requested page served rows already seen. Complete. +STOP_LISTING_ENDED = "listing_ended" +# The page before carried a next-page link, yet the requested page served +# rows already seen: pages were requested and not served. Not complete. +STOP_PAGE_REPEATED = "page_repeated" + + +class PageSequence: + """Which keys each served listing page carried, in page order. + + Measured 2026-10-15 on `/transfers/neuestetransfers/statistik`: the site + answers a page number past the end with the LAST page again, HTTP 200, + its pager still marking the last page as current (`?page=5`, `?page=401` + and `?page=9999` all served page 4's two rows when the listing had four + pages). The default `/statistik/neuestetransfers` view held 20 rows that + day, with no pager, so `?page=2` served page 1 again. Treated as fresh + pages, both runs ended "complete" with pages counted that the site never + served. + + A page repeats when it has rows and every one of them has a key already + seen on an earlier page. Whether that is the listing's end or a lost page + depends on the page before it: no next-page link there means the site + said the listing ends (`listing_ended`); a next-page link there means the + site promised a page and served an old one (`page_repeated`). A page + before it that was not served at all (a concurrent run where it failed) + counts as a promise, the cautious reading. + """ + + def __init__(self): + self._pages: Dict[int, tuple] = {} # page_num -> (frozenset, next_url) + self.repeat: Optional[dict] = None + + def add(self, page_num: int, rows, next_url: Optional[str]) -> Optional[dict]: + """Record one SERVED page (in page order). Returns the verdict dict + the first time a page repeats earlier ones, else None.""" + keys = [getattr(r, "sku", None) for r in rows] + seen = set() + for earlier, (earlier_keys, _next) in self._pages.items(): + if earlier < page_num: + seen |= earlier_keys + repeated = (self.repeat is None and page_num > 1 and keys + and all(k is not None and k in seen for k in keys)) + self._pages[page_num] = (frozenset(k for k in keys if k is not None), + next_url) + if not repeated: + return None + key_set = set(keys) + same_as = next((p for p, (ks, _n) in sorted(self._pages.items()) + if p < page_num and ks == key_set), None) + before = self._pages.get(page_num - 1) + promised = before is None or before[1] is not None + self.repeat = { + "page": page_num, + "same_as": same_as, + "stop_reason": STOP_PAGE_REPEATED if promised else STOP_LISTING_ENDED, + } + logger.info( + "Page %d served %s again (%d row(s), none new)%s.", page_num, + f"page {same_as}" if same_as else "rows from earlier pages", + len(keys), + " although the page before it linked to a next page — recording " + "it as a page not served" if promised else + "; the page before it had no next-page link, so the listing " + "ends there") + return self.repeat + + def counts(self, page_num: int) -> bool: + """False for the repeated page and every page after it: they are not + pages of this listing and are not counted as completed.""" + return self.repeat is None or page_num < self.repeat["page"] + + def meta(self) -> dict: + """Sidecar keys describing the repeat, empty when there was none.""" + if self.repeat is None: + return {} + return {"repeated_page": self.repeat["page"], + "repeat_of_page": self.repeat["same_as"], + "listing_pages": self.repeat["page"] - 1} + + +def settle_repeats(outcomes) -> PageSequence: + """Run PageSequence over finished outcomes in page order — the concurrent + path's version of the check the sequential loop makes as it goes. Pages + that were not served are skipped; the page after one counts it as a + promise (see PageSequence).""" + seq = PageSequence() + for oc in sorted(outcomes, key=lambda o: o.page_num): + if oc.ok and oc.state != "not_found": + if seq.add(oc.page_num, oc.rows, getattr(oc, "next_url", None)): + break + return seq + + # At most one paid solve per page. A challenge that survives a solved token # is not one this run can pass, and a second solve is a second charge for the # same answer. diff --git a/playwright_scraper.py b/playwright_scraper.py index a0ce7f0..1fe7c1a 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -84,6 +84,7 @@ from product_parser import (parse_market_values, parse_club_squad, parse_transfers, parse_player_detail, detect_bot_challenge, page_states_rows, page_url, + next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, MARKET_VALUES_URL, TRANSFERS_URL) @@ -125,6 +126,10 @@ class PageOutcome: # A served page that shows rows the parser could not read: a markup # change, never an empty listing (page_flow.STOP_PARSER_FOUND_NOTHING). parse_failed: bool = False + # The served page's own next-page link (None when it has none): + # decides whether a repeated page is the listing's end or a page + # lost (page_flow.PageSequence). + next_url: Optional[str] = None @property def ok(self) -> bool: @@ -752,6 +757,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.rows = rows outcome.final_url = session.page.url + outcome.next_url = next_page_link(html, outcome.final_url) return outcome @@ -763,10 +769,15 @@ def _worker_pool(pool, worker_index: int): return ProxyPool(proxies[offset:] + proxies[:offset], rotate="per-run") -def _fetch_pages_concurrently(args, pool, specs, concurrency: int): +def _fetch_pages_concurrently(args, pool, specs, concurrency: int, + known_keys=None): """Fetch `specs` [(page_num, url), ...] across `concurrency` workers. Only reached for --mode transfers — see CONCURRENCY_CAPABLE_MODES. + `known_keys` (page 1's row keys) lets a worker stop dispatch once a page + serves nothing new — the site answers a page past the end with an + earlier page. Which page that was, and what it means, is decided + afterwards in page order (page_flow.settle_repeats), not here. """ work = queue.Queue() for spec in specs: @@ -775,6 +786,7 @@ def _fetch_pages_concurrently(args, pool, specs, concurrency: int): results = [] results_lock = threading.Lock() exhausted = threading.Event() + seen_keys = set(known_keys or ()) def worker(index: int): name = f"worker-{index + 1}" @@ -795,7 +807,17 @@ def worker(index: int): page_num, url) with results_lock: results.append(outcome) - if outcome.ok and not outcome.rows: + keys = [getattr(r, "sku", None) for r in outcome.rows] + repeats = bool(outcome.ok and keys and all( + k is not None and k in seen_keys for k in keys)) + if outcome.ok: + seen_keys.update(k for k in keys if k is not None) + if repeats: + logger.info("[%s] page %d served only rows " + "already seen — not dispatching " + "further pages.", name, page_num) + exhausted.set() + elif outcome.ok and not outcome.rows: logger.info("[%s] page %d returned no rows — " "treating that as the end of the " "listing.", name, page_num) @@ -851,6 +873,7 @@ def _plan_or_chain(session, args, first_outcome): def scrape(args) -> int: outcomes: List[PageOutcome] = [] seen_keys = set() + pages = page_flow.PageSequence() blocked = False stop_reason = "single_page_mode" if args.mode not in PAGINATED_MODES else "completed" @@ -900,6 +923,7 @@ def scrape(args) -> int: pass else: seen_keys.update(r.sku for r in first.rows if r.sku is not None) + pages.add(1, first.rows, first.next_url) planned = _plan_or_chain(session, args, first) if args.pages > 1 and concurrency > 1 and planned is None: @@ -916,14 +940,19 @@ def scrape(args) -> int: args.pages, concurrency, f" over {len(pool)} exit(s)" if pool else "") rest, unattempted, exhausted = _fetch_pages_concurrently( - args, pool, specs, concurrency) + args, pool, specs, concurrency, + known_keys=[r.sku for r in first.rows]) outcomes.extend(rest) + pages = page_flow.settle_repeats(outcomes) - failed = [o for o in rest if not o.ok] + failed = [o for o in rest + if not o.ok and pages.counts(o.page_num)] if failed: worst = min(failed, key=lambda o: o.page_num) stop_reason = page_flow.failure_reason(worst) - blocked = any(o.blocked_by for o in rest) + blocked = any(o.blocked_by for o in failed) + elif pages.repeat: + stop_reason = pages.repeat["stop_reason"] elif exhausted: stop_reason = "no_new_products" elif unattempted: @@ -954,6 +983,12 @@ def scrape(args) -> int: blocked = outcome.blocked_by is not None break + # A page that only repeats earlier rows is not a fresh page: + # the listing's end, or a page the site did not serve. + repeat = pages.add(page_num, outcome.rows, outcome.next_url) + if repeat: + stop_reason = repeat["stop_reason"] + break fresh_count = sum(1 for r in outcome.rows if r.sku is None or r.sku not in seen_keys) seen_keys.update(r.sku for r in outcome.rows @@ -984,15 +1019,20 @@ def scrape(args) -> int: all_rows = [] merged_seen = set() + counted = [o for o in outcomes if pages.counts(o.page_num)] for oc in sorted(outcomes, key=lambda o: o.page_num): + if not pages.counts(oc.page_num): + logger.info("Page %d: not counted — it repeated an earlier " + "page (%d row(s) dropped).", oc.page_num, len(oc.rows)) + continue fresh = dedupe_by_key(oc.rows, merged_seen, key="sku") if len(fresh) < len(oc.rows): logger.info("Page %d: dropped %d duplicate row(s).", oc.page_num, len(oc.rows) - len(fresh)) all_rows.extend(fresh) - ok_pages = [o for o in outcomes if o.ok] - failed_pages = [o.page_num for o in outcomes if not o.ok] + ok_pages = [o for o in counted if o.ok] + failed_pages = [o.page_num for o in counted if not o.ok] final_url = (max(ok_pages, key=lambda o: o.page_num).final_url if ok_pages else args.url) @@ -1001,7 +1041,8 @@ def scrape(args) -> int: pages_requested=args.pages, pages_completed=len(ok_pages), pages_failed=failed_pages, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", - start_url=args.url, final_url=final_url) + start_url=args.url, final_url=final_url, + extra=pages.meta()) def parse_args(): diff --git a/product_parser.py b/product_parser.py index c5ff7be..d3ed33f 100644 --- a/product_parser.py +++ b/product_parser.py @@ -805,6 +805,22 @@ def is_not_found_page(html: Optional[str], status: Optional[int] = None) -> bool return status in NOT_FOUND_STATUSES or bool(html and _NOT_FOUND_RE.search(html)) +# The edge's bare nginx 5xx page. Captured 2026-10-15 twice in one minute +# (`502 Bad Gateway`, 155 bytes): once as the answer to a +# listing page, once as the document left after the AWS WAF challenge +# cleared. In the second case the challenge's 202 had been discarded, as it +# must be, so nothing but the body said the page had failed; it classified as +# `empty`, and the run carried on and ended "complete" with page 1 missing. +# Read from the body, it classifies as the same failure the status already +# gave it where the status was visible. +_GATEWAY_ERROR_RE = re.compile(r"\s*5\d\d [^<]{0,40}\s*\s*" + r"\s*

\s*5\d\d", re.IGNORECASE) + + +def is_gateway_error_page(html: Optional[str]) -> bool: + return bool(html and len(html) < 4096 and _GATEWAY_ERROR_RE.search(html)) + + _CANONICAL_RES = (re.compile(r' Optional[str]: return None +def next_page_link(html: Optional[str], base_url: str) -> Optional[str]: + """The page's own next-page link, absolute, or None when it has none. + + Read from the served HTML so every engine, the Scraper API client + included, records the same answer for the same page. A listing that fits + on one page carries neither `` nor a pager (measured on + the latest-transfers listing 2026-10-15, 20 rows).""" + if not html: + return None + el = BeautifulSoup(html, "html.parser").select_one( + "link[rel='next'], a[rel='next']") + href = el.get("href") if el else None + return urljoin(base_url, href) if href else None + + def page_states_rows(html: Optional[str], mode: str) -> bool: """Whether the page itself shows rows the parser should have read. @@ -905,6 +936,9 @@ def detect_page_state(html: str, status: Optional[int] = None, if is_not_found_page(html, status): return "not_found" + if is_gateway_error_page(html): + return "blocked" + vendor = detect_bot_challenge(html, url=url) if vendor: return "captcha" if vendor in SOLVABLE_VENDORS else "blocked" diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index ebfa259..38e880d 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -56,6 +56,7 @@ from product_parser import (parse_market_values, parse_club_squad, parse_transfers, parse_player_detail, detect_bot_challenge, page_states_rows, page_url, + next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, MARKET_VALUES_URL, TRANSFERS_URL) @@ -146,6 +147,10 @@ class PageOutcome: # A served page that shows rows the parser could not read: a markup # change, never an empty listing (page_flow.STOP_PARSER_FOUND_NOTHING). parse_failed: bool = False + # The served page's own next-page link (None when it has none): + # decides whether a repeated page is the listing's end or a page + # lost (page_flow.PageSequence). + next_url: Optional[str] = None @property def ok(self) -> bool: @@ -599,12 +604,14 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.rows = rows outcome.final_url = page.url + outcome.next_url = next_page_link(html, outcome.final_url) return outcome def scrape(args) -> int: outcomes: List[PageOutcome] = [] seen_keys = set() + pages = page_flow.PageSequence() blocked = False stop_reason = "single_page_mode" if args.mode not in PAGINATED_MODES else "completed" @@ -633,6 +640,7 @@ def scrape(args) -> int: pass else: seen_keys.update(r.sku for r in first.rows if r.sku is not None) + pages.add(1, first.rows, first.next_url) site_next = _next_page_href(session) addressable = page_flow.pagination_is_addressable( args.mode, first.final_url, site_next) @@ -658,6 +666,12 @@ def scrape(args) -> int: blocked = outcome.blocked_by is not None break + # A page that only repeats earlier rows is not a fresh page: + # the listing's end, or a page the site did not serve. + repeat = pages.add(page_num, outcome.rows, outcome.next_url) + if repeat: + stop_reason = repeat["stop_reason"] + break fresh_count = sum(1 for r in outcome.rows if r.sku is None or r.sku not in seen_keys) seen_keys.update(r.sku for r in outcome.rows if r.sku is not None) @@ -685,15 +699,20 @@ def scrape(args) -> int: all_rows = [] merged_seen = set() + counted = [o for o in outcomes if pages.counts(o.page_num)] for oc in sorted(outcomes, key=lambda o: o.page_num): + if not pages.counts(oc.page_num): + logger.info("Page %d: not counted — it repeated an earlier " + "page (%d row(s) dropped).", oc.page_num, len(oc.rows)) + continue fresh = dedupe_by_key(oc.rows, merged_seen, key="sku") if len(fresh) < len(oc.rows): logger.info("Page %d: dropped %d duplicate row(s).", oc.page_num, len(oc.rows) - len(fresh)) all_rows.extend(fresh) - ok_pages = [o for o in outcomes if o.ok] - failed_pages = [o.page_num for o in outcomes if not o.ok] + ok_pages = [o for o in counted if o.ok] + failed_pages = [o.page_num for o in counted if not o.ok] final_url = (max(ok_pages, key=lambda o: o.page_num).final_url if ok_pages else args.url) @@ -702,7 +721,8 @@ def scrape(args) -> int: pages_requested=args.pages, pages_completed=len(ok_pages), pages_failed=failed_pages, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", - start_url=args.url, final_url=final_url) + start_url=args.url, final_url=final_url, + extra=pages.meta()) def parse_args(): diff --git a/scraper_api_client.py b/scraper_api_client.py index 4cded4b..2e3884b 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -132,7 +132,7 @@ from product_parser import (parse_market_values, parse_club_squad, parse_transfers, parse_player_detail, detect_bot_challenge, detect_page_state, page_url, - page_states_rows, canonical_url, + page_states_rows, canonical_url, next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, MARKET_VALUES_URL, TRANSFERS_URL) @@ -433,6 +433,7 @@ def scrape(args) -> int: pages_completed = 0 pages_failed: List[int] = [] final_url = args.url + pages = page_flow.PageSequence() rows, blocked_by, _status, page1_html, failure = _fetch_one_page(args, 1, args.url) if blocked_by is not None: @@ -449,6 +450,7 @@ def scrape(args) -> int: pages_completed = 1 fresh = dedupe_by_key(rows, seen_keys, key="sku") all_rows.extend(fresh) + pages.add(1, rows, next_page_link(page1_html, args.url)) addressable = True if args.pages > 1 and args.mode in PAGINATED_MODES: @@ -485,7 +487,7 @@ def scrape(args) -> int: elif args.pages > 1: for page_num in range(2, args.pages + 1): url = page_url(args.url, page_num) - page_rows, page_blocked_by, _status, _html, failure = _fetch_one_page(args, page_num, url) + page_rows, page_blocked_by, _status, page_html, failure = _fetch_one_page(args, page_num, url) if page_blocked_by is not None: stop_reason = f"blocked_{page_blocked_by}" blocked = True @@ -496,6 +498,17 @@ def scrape(args) -> int: else failure) pages_failed.append(page_num) break + # A page that only repeats earlier rows is not a fresh page: + # the listing's end, or a page the site did not serve. Not + # counted as completed either way (page_flow.PageSequence). + repeat = pages.add(page_num, page_rows, + next_page_link(page_html, url)) + if repeat: + logger.info("Page %d: not counted — it repeated an " + "earlier page (%d row(s) dropped).", + page_num, len(page_rows)) + stop_reason = repeat["stop_reason"] + break pages_completed += 1 final_url = url fresh_count = sum(1 for r in page_rows @@ -531,7 +544,8 @@ def scrape(args) -> int: pages_requested=args.pages, pages_completed=pages_completed, pages_failed=pages_failed, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", - start_url=args.url, final_url=final_url) + start_url=args.url, final_url=final_url, + extra=pages.meta()) def main() -> int: diff --git a/selenium_scraper.py b/selenium_scraper.py index 6586177..fa6e08e 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -54,6 +54,7 @@ from product_parser import (parse_market_values, parse_club_squad, parse_transfers, parse_player_detail, detect_bot_challenge, page_states_rows, page_url, + next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, MARKET_VALUES_URL, TRANSFERS_URL) @@ -117,6 +118,10 @@ class PageOutcome: # A served page that shows rows the parser could not read: a markup # change, never an empty listing (page_flow.STOP_PARSER_FOUND_NOTHING). parse_failed: bool = False + # The served page's own next-page link (None when it has none): + # decides whether a repeated page is the listing's end or a page + # lost (page_flow.PageSequence). + next_url: Optional[str] = None @property def ok(self) -> bool: @@ -544,12 +549,14 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.rows = rows outcome.final_url = final_url + outcome.next_url = next_page_link(html, outcome.final_url) return outcome def scrape(args) -> int: outcomes: List[PageOutcome] = [] seen_keys = set() + pages = page_flow.PageSequence() blocked = False stop_reason = "single_page_mode" if args.mode not in PAGINATED_MODES else "completed" @@ -577,6 +584,7 @@ def scrape(args) -> int: pass else: seen_keys.update(r.sku for r in first.rows if r.sku is not None) + pages.add(1, first.rows, first.next_url) site_next = _next_page_href(session) addressable = page_flow.pagination_is_addressable( args.mode, first.final_url, site_next) @@ -602,6 +610,12 @@ def scrape(args) -> int: blocked = outcome.blocked_by is not None break + # A page that only repeats earlier rows is not a fresh page: + # the listing's end, or a page the site did not serve. + repeat = pages.add(page_num, outcome.rows, outcome.next_url) + if repeat: + stop_reason = repeat["stop_reason"] + break fresh_count = sum(1 for r in outcome.rows if r.sku is None or r.sku not in seen_keys) seen_keys.update(r.sku for r in outcome.rows if r.sku is not None) @@ -628,15 +642,20 @@ def scrape(args) -> int: all_rows = [] merged_seen = set() + counted = [o for o in outcomes if pages.counts(o.page_num)] for oc in sorted(outcomes, key=lambda o: o.page_num): + if not pages.counts(oc.page_num): + logger.info("Page %d: not counted — it repeated an earlier " + "page (%d row(s) dropped).", oc.page_num, len(oc.rows)) + continue fresh = dedupe_by_key(oc.rows, merged_seen, key="sku") if len(fresh) < len(oc.rows): logger.info("Page %d: dropped %d duplicate row(s).", oc.page_num, len(oc.rows) - len(fresh)) all_rows.extend(fresh) - ok_pages = [o for o in outcomes if o.ok] - failed_pages = [o.page_num for o in outcomes if not o.ok] + ok_pages = [o for o in counted if o.ok] + failed_pages = [o.page_num for o in counted if not o.ok] final_url = (max(ok_pages, key=lambda o: o.page_num).final_url if ok_pages else args.url) @@ -645,7 +664,8 @@ def scrape(args) -> int: pages_requested=args.pages, pages_completed=len(ok_pages), pages_failed=failed_pages, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", - start_url=args.url, final_url=final_url) + start_url=args.url, final_url=final_url, + extra=pages.meta()) def parse_args(): diff --git a/smoke_test.py b/smoke_test.py index 625905a..e25e61a 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -4148,6 +4148,35 @@ def test_retries_is_validated_in_every_entry_point(): """ +# Real captures of the latest-transfers listing, 2026-10-15 (Playwright, +# system Chromium, no proxy). Cut to the grid's first rows, its pager, the +# head's rel=next/prev and the grid's keys div; scripts, ads and consent +# markup removed. Nothing else edited. +# +# What they show: the default view applies the site's own market-value floor +# (its filter form and tab links carry minMarktwert=500000). That day it held +# 20 transfers, under one 25-row page, so it had no pager and no next link, +# and ?page=2 served page 1 again. Without the floor the same listing had +# four pages (77 rows), and any page past the fourth served the fourth again. +# `/statistik/neuestetransfers`, the site's default view +FIX_TRANSFERS_ONE_PAGE_P1 = """Latest transfers | Transfermarkt
PlayerAgeNat.LeftJoinedFee
Hernán López MuñozHernán López Muñoz
Attacking Midfield
26Argentina
San Jose EarthquakesSJ Earthquakes
United States United States
AA Argentinos JuniorsArgentinos Jrs.
Argentina Argentina
€2.00m
Anton TangheAnton Tanghe
Centre-Back
27Belgium
Without ClubWithout Club
FK Kauno ŽalgirisKauno Žalgiris
Lithuania Lithuania
-
Mark McKenzieMark McKenzie
Centre-Back
27United States
Jamaica
FC ToulouseToulouse
FranceLigue 1
Philadelphia UnionPhiladelphia
United States United States
?
""" + +# `/statistik/neuestetransfers?page=2`, same session: page 1 again +FIX_TRANSFERS_ONE_PAGE_P2 = """Latest transfers | Page 2 | Transfermarkt
PlayerAgeNat.LeftJoinedFee
Hernán López MuñozHernán López Muñoz
Attacking Midfield
26Argentina
San Jose EarthquakesSJ Earthquakes
United States United States
AA Argentinos JuniorsArgentinos Jrs.
Argentina Argentina
€2.00m
Anton TangheAnton Tanghe
Centre-Back
27Belgium
Without ClubWithout Club
FK Kauno ŽalgirisKauno Žalgiris
Lithuania Lithuania
-
Mark McKenzieMark McKenzie
Centre-Back
27United States
Jamaica
FC ToulouseToulouse
FranceLigue 1
Philadelphia UnionPhiladelphia
United States United States
?
""" + +# the same listing with `minMarktwert=0`, page 1 +FIX_TRANSFERS_PAGED_P1 = """Latest transfers | Transfermarkt
PlayerAgeNat.LeftJoinedFee
Amir MusaevAmir Musaev
Central Midfield
18Russia
Zenit St. Petersburg IIZenit II
Russia Russia
Zenit St. PetersburgZenit S-Pb
RussiaPremier Liga
-
Ivan YakovlevIvan Yakovlev
Centre-Forward
33Russia
FC Irkutsk Irkutsk
Russia Russia
Without ClubWithout Club
-
Timothy SyrelTimothy Syrel
Goalkeeper
22United States
Belarus
Bumprom GomelBumprom
Belarus Belarus
Without ClubWithout Club
-
""" + +# ...page 2 +FIX_TRANSFERS_PAGED_P2 = """Latest transfers | Page 2 | Transfermarkt
PlayerAgeNat.LeftJoinedFee
Yu KobayashiYu Kobayashi
Centre-Forward
39Japan
Kawasaki FrontaleKawasaki Front.
JapanJ1 League
Retired Retired
-
Anders Bjørntvedt OlsenAnders Bjørntvedt Olsen
Right Winger
27Norway
Lyn 1896 FKLyn
Norway Norway
Stabæk FotballStabæk
Norway Norway
free transfer
Roberto BochiRoberto Bochi
Defensive Midfield
39Argentina
Italy
CA AldosiviAldosivi
Argentina Argentina
Without ClubWithout Club
-
""" + +# ...the last page (4 of 4, 2 rows; requested as ?page=399) +FIX_TRANSFERS_PAGED_LAST = """Latest transfers | Page 399 | Transfermarkt
PlayerAgeNat.LeftJoinedFee
Idzham EszuanIdzham Eszuan
Centre-Back
19Singapore
Military Service (Singapore)Military
Singapore Singapore
Lion City Sailors ReservesLС Sailors Res
SingaporePremier League 2
-
Kosuke MatsumuraKosuke Matsumura
Defensive Midfield
22Japan
Hosei UniversityHosei Univ.
Japan Japan
Yokohama F. MarinosYokohama F. M.
JapanJ1 League
free transfer
""" + +# ...?page=401: the last page again, HTTP 200 +FIX_TRANSFERS_PAGED_PAST_END = """Latest transfers | Page 401 | Transfermarkt
PlayerAgeNat.LeftJoinedFee
Idzham EszuanIdzham Eszuan
Centre-Back
19Singapore
Military Service (Singapore)Military
Singapore Singapore
Lion City Sailors ReservesLС Sailors Res
SingaporePremier League 2
-
Kosuke MatsumuraKosuke Matsumura
Defensive Midfield
22Japan
Hosei UniversityHosei Univ.
Japan Japan
Yokohama F. MarinosYokohama F. M.
JapanJ1 League
free transfer
""" + + def test_atomic_writes_and_modes(): group("output: a failed write keeps the previous file; modes follow open()") ok = True @@ -4650,6 +4679,272 @@ def test_no_private_citations(): return ok +# The edge's nginx 502, exactly as saved on 2026-10-15 for page 1 of the +# unfiltered listing, after the AWS WAF challenge had cleared (so no status +# was left to judge it by). +FIX_NGINX_502 = """502 Bad Gateway + +

502 Bad Gateway

+
nginx/1.18.0
+ + + + + + + + +""" + + +TRANSFERS_PAGED_URL = ("https://www.transfermarkt.com/transfers/neuestetransfers/" + "statistik?land_id=0&verein_land_id=0&wettbewerb_id=alle" + "&minMarktwert=0&maxMarktwert=500000000") + + +class _ListingPage: + """The bit of a Playwright page _plan_or_chain reads: the next link.""" + + def __init__(self): + self.html, self.url = "", "" + + def query_selector(self, _selector): + from product_parser import next_page_link + href = next_page_link(self.html, self.url or "https://www.transfermarkt.com/") + if not href: + return None + return type("El", (), {"get_attribute": lambda self, _n: href})() + + +class _ListingSession(_FakeSession): + def __init__(self, pool=None): + super().__init__(pool) + self.page = _ListingPage() + + def relaunch(self): + return self + + +def _run_listing(modname, url, pages, html_for_page, extra_argv=()): + """Run one entry point's real scrape() for --mode transfers with the + fetch replaced by real captures. Returns (rc, meta, row skus).""" + from product_parser import next_page_link, page_number_from_url + mod = importlib.import_module(modname) + tmp = tempfile.mkdtemp(prefix="tm_listing_") + out = os.path.join(tmp, "out") + saved = {} + saved_argv = sys.argv + # Every variable env_config reads is cleared for the run: one left behind + # by an earlier check (a CDP endpoint, a proxy) changes which path runs. + env_names = set(env_config.ENV_KEYS) | {"TWOCAPTCHA_KEY"} + saved_env = {k: os.environ.get(k) for k in env_names} + + def patch(name, value): + saved[name] = getattr(mod, name) + setattr(mod, name, value) + + try: + for k in env_names: + os.environ.pop(k, None) + if modname == "scraper_api_client": + # Its parse_args requires a key; the fetch below is stubbed, so + # nothing is ever sent anywhere. + os.environ["TWOCAPTCHA_KEY"] = "test-placeholder" + sys.argv = [modname + ".py", "--mode", "transfers", "--url", url, + "--pages", str(pages), "--delay", "0", "--out", out, + "--format", "json"] + list(extra_argv) + args = mod.parse_args() + if modname == "scraper_api_client": + patch("fetch_html", lambda a, u, timeout: + (html_for_page(page_number_from_url(u)), 200, {})) + else: + session = _ListingSession() + + def fake_fetch(sess, a, pool, page_num, u): + html = html_for_page(page_number_from_url(u)) + if hasattr(sess, "page"): + sess.page.html, sess.page.url = html, u + session.page.html, session.page.url = html, u + oc = mod.PageOutcome(page_num=page_num, url=u, final_url=u, + state="content") + oc.rows = parse_transfers(html, u, page_num=page_num) + oc.next_url = next_page_link(html, u) + return oc + + patch("_fetch_one_page", fake_fetch) + if modname == "playwright_scraper": + patch("sync_playwright", lambda: _FakePlaywright()) + patch("_BrowserSession", lambda pw, a, pool, **kw: session) + else: + patch("_Session", lambda *a, **kw: session) + patch("_next_page_href", lambda sess: next_page_link( + session.page.html, session.page.url)) + if modname == "puppeteer_scraper": + patch("_AsyncBridge", lambda: type( + "B", (), {"close": lambda self: None})()) + rc = mod.scrape(args) + finally: + for name, value in saved.items(): + setattr(mod, name, value) + sys.argv = saved_argv + for k, v in saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + meta = {} + if os.path.exists(out + ".meta.json"): + meta = json.load(open(out + ".meta.json", encoding="utf-8")) + skus = [] + if os.path.exists(out + ".json"): + skus = [r["sku"] for r in json.load(open(out + ".json", encoding="utf-8"))] + shutil.rmtree(tmp, ignore_errors=True) + return rc, meta, skus + + +def test_listing_end_and_repeated_pages(skips): + group("latest transfers: a page that repeats an earlier one (real captures, 2026-10-15)") + from product_parser import next_page_link + ok = True + + # The captures themselves. + one1 = parse_transfers(FIX_TRANSFERS_ONE_PAGE_P1, TRANSFERS_URL) + one2 = parse_transfers(FIX_TRANSFERS_ONE_PAGE_P2, TRANSFERS_URL + "?page=2", page_num=2) + ok &= check("default view: the rows read with their values", + [(r.sku, r.title, r.player_id, r.age) for r in one1] == [ + ("6640068", "Hernán López Muñoz", "665309", 26), + ("6633411", "Anton Tanghe", "460848", 27), + ("6622615", "Mark McKenzie", "429259", 27)]) + ok &= check("default view: the disclosed fee is EUR 2,000,000", + (one1[0].price, one1[0].currency, one1[0].fee_type) + == (2000000.0, "EUR", "disclosed")) + ok &= check("default view: an undisclosed fee stays null", + (one1[2].price, one1[2].fee_type) == (None, "undisclosed")) + ok &= check("default view: no next link on page 1", + next_page_link(FIX_TRANSFERS_ONE_PAGE_P1, TRANSFERS_URL) is None) + ok &= check("default view: ?page=2 served page 1's rows again", + [r.sku for r in one2] == [r.sku for r in one1]) + p1 = parse_transfers(FIX_TRANSFERS_PAGED_P1, TRANSFERS_PAGED_URL) + p2 = parse_transfers(FIX_TRANSFERS_PAGED_P2, TRANSFERS_PAGED_URL, page_num=2) + last = parse_transfers(FIX_TRANSFERS_PAGED_LAST, TRANSFERS_PAGED_URL, page_num=4) + ok &= check("unfiltered view: page 1's next link is ?page=2 with the filters kept", + next_page_link(FIX_TRANSFERS_PAGED_P1, TRANSFERS_PAGED_URL) + == TRANSFERS_PAGED_URL + "&page=2") + ok &= check("unfiltered view: page 1 and page 2 share no transfer id", + not {r.sku for r in p1} & {r.sku for r in p2} + and [r.sku for r in p2] == ["6609682", "6593365", "6587122"]) + ok &= check("unfiltered view: the constructed ?page=2 matches the site's own link", + page_flow.pagination_is_addressable( + "transfers", TRANSFERS_PAGED_URL, + next_page_link(FIX_TRANSFERS_PAGED_P1, TRANSFERS_PAGED_URL))) + ok &= check("unfiltered view: the last page has no next link", + next_page_link(FIX_TRANSFERS_PAGED_LAST, TRANSFERS_PAGED_URL) is None + and [r.sku for r in last] == ["5605112", "5520006"]) + + # The rule, on its own. + seq = page_flow.PageSequence() + seq.add(1, one1, None) + verdict = seq.add(2, one2, None) + ok &= check("page 2 repeating a page with no next link ends the listing", + verdict and verdict["stop_reason"] == page_flow.STOP_LISTING_ENDED + and verdict["same_as"] == 1) + ok &= check("the repeated page is not counted", + seq.counts(1) and not seq.counts(2) and not seq.counts(3)) + ok &= check("the sidecar says which page repeated which", + seq.meta() == {"repeated_page": 2, "repeat_of_page": 1, + "listing_pages": 1}) + seq = page_flow.PageSequence() + seq.add(1, p1, "next") + verdict = seq.add(2, p1, None) + ok &= check("a repeat after a page that linked onward is a page not served", + verdict and verdict["stop_reason"] == page_flow.STOP_PAGE_REPEATED) + ok &= check("page_repeated is not complete; listing_ended is", + page_flow.STOP_PAGE_REPEATED not in COMPLETE_STOP_REASONS + and page_flow.STOP_LISTING_ENDED in COMPLETE_STOP_REASONS) + seq = page_flow.PageSequence() + seq.add(1, p1, "next") + ok &= check("a page with new rows is not a repeat", seq.add(2, p2, "next") is None) + ok &= check("an empty page is not a repeat (the empty-page stop handles it)", + seq.add(3, [], None) is None) + + # A bare nginx 5xx is a page not served, with or without its status. + from product_parser import detect_page_state, is_gateway_error_page + ok &= check("the edge's 502 page, with no status, is not an empty page", + detect_page_state(FIX_NGINX_502) == "blocked") + ok &= check("...and classifies the same with its status", + detect_page_state(FIX_NGINX_502, status=502) == "blocked") + ok &= check("the 502 marker matches none of the served or 404 captures", + not any(is_gateway_error_page(h) for h in ( + FIX_TRANSFERS_ONE_PAGE_P1, FIX_TRANSFERS_PAGED_P1, + FIX_TRANSFERS_PAGED_LAST, FIX_MARKET_VALUES, FIX_CLUB_SQUAD, + FIX_TRANSFERS, FIX_PLAYER_DETAIL, FIX_NOT_FOUND, + FIX_NGINX_404, FIX_AWS_WAF_CHALLENGE))) + gateway_first = lambda n: FIX_NGINX_502 if n == 1 else FIX_TRANSFERS_PAGED_P2 + rc, meta, skus = _run_listing("scraper_api_client", TRANSFERS_PAGED_URL, 2, + gateway_first) + ok &= check("a 502 for page 1 is not a complete run (got rc=%s %s)" + % (rc, meta.get("status")), + rc != 0 and meta.get("status") != "complete") + + # The runs below replace each engine's _fetch_one_page, so whether the + # real one records the page's next link is checked on its source. + for name in ENGINE_FILES: + tree = ast.parse(open(os.path.join(REPO_ROOT, name), encoding="utf-8").read()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) + and n.name == "_fetch_one_page") + sets_next = any( + isinstance(n, ast.Assign) + and ast.unparse(n.targets[0]) == "outcome.next_url" + and isinstance(n.value, ast.Call) + and ast.unparse(n.value.func) == "next_page_link" + for n in ast.walk(fn)) + ok &= check("%s: _fetch_one_page records the served page's next link" + % name, sets_next) + + # Each entry point, through its real scrape(), fed the captures. + one_page = lambda n: FIX_TRANSFERS_ONE_PAGE_P1 if n == 1 else FIX_TRANSFERS_ONE_PAGE_P2 + paged = {1: FIX_TRANSFERS_PAGED_P1, 2: FIX_TRANSFERS_PAGED_P2, + 3: FIX_TRANSFERS_PAGED_LAST} + four = lambda n: paged.get(n, FIX_TRANSFERS_PAGED_PAST_END) + planted = lambda n: FIX_TRANSFERS_PAGED_P1 + want_paged = ([r.sku for r in p1] + [r.sku for r in p2] + [r.sku for r in last]) + runs = [(m, ()) for m in ("scraper_api_client", "playwright_scraper", + "puppeteer_scraper", "selenium_scraper")] + runs.append(("playwright_scraper", ("--concurrency", "3"))) + for modname, extra in runs: + label = modname + (" --concurrency 3" if extra else "") + try: + rc, meta, skus = _run_listing(modname, TRANSFERS_URL, 3, one_page, extra) + except ImportError as e: + skips.append("%s repeated-page run (%s)" % (label, e)) + continue + ok &= check("%s: one-page listing, --pages 3 -> exit 0, listing_ended, " + "1 page counted, 3 rows (got rc=%s %s %s %s)" + % (label, rc, meta.get("stop_reason"), meta.get("pages_completed"), len(skus)), + rc == 0 and meta.get("status") == "complete" + and meta.get("stop_reason") == "listing_ended" + and meta.get("pages_completed") == 1 + and meta.get("repeated_page") == 2 and meta.get("repeat_of_page") == 1 + and skus == [r.sku for r in one1]) + rc, meta, skus = _run_listing(modname, TRANSFERS_PAGED_URL, 6, four, extra) + ok &= check("%s: 3-page listing, --pages 6 -> exit 0, 3 pages counted, " + "8 rows in page order, page 4 recorded as a repeat of 3 " + "(got rc=%s %s %s %s)" + % (label, rc, meta.get("stop_reason"), meta.get("pages_completed"), skus), + rc == 0 and meta.get("stop_reason") == "listing_ended" + and meta.get("pages_completed") == 3 + and meta.get("repeated_page") == 4 and meta.get("repeat_of_page") == 3 + and skus == want_paged) + rc, meta, skus = _run_listing(modname, TRANSFERS_PAGED_URL, 3, planted, extra) + ok &= check("%s: page 1 links to page 2 but page 2 serves page 1 -> " + "exit 6, page_repeated, partial (got rc=%s %s %s)" + % (label, rc, meta.get("stop_reason"), meta.get("status")), + rc == 6 and meta.get("stop_reason") == "page_repeated" + and meta.get("status") == "partial" + and meta.get("pages_completed") == 1 and skus == [r.sku for r in p1]) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -4712,6 +5007,7 @@ def main() -> int: ok &= test_solve_budget_is_per_page() ok &= test_credential_scan_judges_paths_inside_the_repo() ok &= test_no_private_citations() + ok &= test_listing_end_and_repeated_pages(skips) print() if _failures: From 360575a1b6f5e6a8af3c5c70f300f6b87fe26a8c Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Fri, 16 Oct 2026 06:12:06 +0000 Subject: [PATCH 2/2] Add --min-value for the transfers listing; retry a site 5xx instead of calling it blocked --mode transfers --min-value EUR (0 = no floor) reads the latest-transfers listing through its filtered path with minMarktwert set, keeping any filters a typed --url carries. Without the flag the site's own default view (filtered by the site to EUR 500,000+) is read as before. The sidecar records min_market_value_eur and market_value_filter (url / site_default). Negative, non-integer, or used with another --mode: exit 2. All four entry points. The canary's transfers run now uses --min-value 0 and checks the recorded floor. An HTTP 5xx from the site, or the edge's bare nginx 5xx page when no status is visible, is now a server error, not a block: the page is fetched again on the same exit up to 2 times (--retry-delay x1, x2; no rotation, no paid solve, under the page's one solve budget), then failed as target_http_5xx, exit 5 on page 1 and 6 later. A visible 5xx used to end exit 3. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/canary.yml | 6 +- CHANGELOG.md | 30 ++- README.md | 19 +- TROUBLESHOOTING.md | 12 +- page_flow.py | 77 ++++++++ playwright_scraper.py | 44 ++++- product_parser.py | 57 +++++- puppeteer_scraper.py | 44 ++++- scraper_api_client.py | 33 +++- selenium_scraper.py | 44 ++++- smoke_test.py | 350 +++++++++++++++++++++++++++++++++-- 11 files changed, 671 insertions(+), 45 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index ab3bb60..f302dd9 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -118,7 +118,7 @@ jobs: set +e python3 playwright_scraper.py \ --mode transfers --pages 3 \ - --url "https://www.transfermarkt.com/transfers/neuestetransfers/statistik?land_id=0&verein_land_id=0&wettbewerb_id=alle&minMarktwert=0&maxMarktwert=500000000" \ + --min-value 0 \ --out canary_transfers --dump-html canary_transfers.html echo "exit_code=$?" >> "$GITHUB_OUTPUT" @@ -252,6 +252,10 @@ jobs: meta = json.loads(Path("canary_transfers.meta.json").read_text()) fail = [] + if meta.get("min_market_value_eur") != 0: + fail.append(f"min_market_value_eur={meta.get('min_market_value_eur')!r}, " + f"expected 0 from --min-value 0") + # Tests the pagination and the parser, not the transfer season: # more than one page's worth of rows (25 a page) means a second # page was followed, whatever the listing's depth that day. diff --git a/CHANGELOG.md b/CHANGELOG.md index 961faa1..f3cd902 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,8 +43,27 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). > `no_new_products`) with `repeated_page`, `repeat_of_page` and > `listing_pages` in the sidecar; when the page before it linked onward the > run ends `stop_reason: page_repeated`, exit 6 (was exit 0, "complete"). -> `pages_completed` no longer counts the repeated page. The edge's bare nginx -> 5xx page is now a failed page in every engine, never an empty one. +> `pages_completed` no longer counts the repeated page. +> +> **Behaviour change (server errors):** an HTTP 5xx from Transfermarkt, or +> the edge's bare nginx 5xx page where no status is visible, is a server +> error, not a block. The page is fetched again on the same exit up to 2 +> times (`--retry-delay` x1, x2, no proxy rotation, no paid solve), then +> failed with `stop_reason: target_http_5xx`: exit 5 on page 1, exit 6 on a +> later page. A visible 5xx used to be exit 3 ("blocked"), and the bare 502 +> page used to read as an empty page. + +### Added + +- **`--mode transfers --min-value EUR`** (all four entry points) reads the + latest-transfers listing filtered to players valued at least EUR (0 = no + floor) through its filtered path, + `/transfers/neuestetransfers/statistik?...&minMarktwert=EUR`, keeping any + filters a typed `--url` already carries. Without it, the site's own default + view is read, as before. A negative or non-integer value, or the flag with + another `--mode`, is exit 2. The sidecar records `min_market_value_eur` + and `market_value_filter` (`url`, or `site_default` when the floor is the + site's own and not in the URL). ### Fixed @@ -58,14 +77,15 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). listing's end (when the page before it had no next link) or as a page not served (when it had one), in all four entry points and in Playwright's `--concurrency` path, which also stops dispatching once a page repeats. - The README documents the filter and the unfiltered URL (4 pages, 77 - transfers that day). + The README documents the filter; `--min-value 0` reads the unfiltered + listing (4 pages, 77 transfers that day). - **A 502 after the AWS WAF challenge ended a run "complete" with page 1 missing.** Measured 2026-10-15: the challenge cleared and the document left in the browser was the edge's 155-byte nginx `502 Bad Gateway`. Its status had rightly been discarded with the challenge's, so it classified as an empty page, and the run carried on to pages 2-5 and reported 52 of 77 - rows as complete. The page is now recognised from its body. + rows as complete. The page is now recognised from its body as a server + error (see above). - **A failed write destroyed the previous good output.** `write_json`, `write_csv`, the `.meta.json` sidecar and `diff_runs.py --out` all opened their target with `open(path, "w")`, which truncates first: a dump that diff --git a/README.md b/README.md index 7923594..1b31d80 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,12 @@ python3 playwright_scraper.py --mode club-squad --club-id 281 --season 2025 # The latest-transfers ledger, up to 3 pages (safe to run with --concurrency). # The default view is filtered to players valued EUR 500k+ and can fit on one -# page; see "How deep that listing goes" below for the unfiltered URL. +# page; see "How deep that listing goes" below. python3 playwright_scraper.py --mode transfers --pages 3 --concurrency 3 +# The same listing with no market-value floor (0 = unfiltered) +python3 playwright_scraper.py --mode transfers --min-value 0 --pages 6 + # One player's full profile python3 playwright_scraper.py --mode player --player-id 418560 ``` @@ -310,8 +313,12 @@ see.** Measured 2026-10-15: a cap on what this scraper reads. - Without the floor, the same listing had **4 pages, 77 transfers** (25, 25, 25, 2), each page's next link a plain `&page=N` with the filters kept. - Pass that URL to read it: - `--mode transfers --url "https://www.transfermarkt.com/transfers/neuestetransfers/statistik?land_id=0&verein_land_id=0&wettbewerb_id=alle&minMarktwert=0&maxMarktwert=500000000" --pages 6`. + `--min-value 0` reads it (`--mode transfers --min-value 0 --pages 6`); + `--min-value 250000` sets another floor. The flag builds + `/transfers/neuestetransfers/statistik?...&minMarktwert=EUR`, keeps any + filters a typed `--url` carries, and the sidecar records + `min_market_value_eur` (`null` with `market_value_filter: site_default` + when the run read the site's default view, whose floor is not in its URL). - A page number past the end is answered **HTTP 200 with the last page again**: `?page=2` of the 20-row view served page 1, and `?page=5`, `?page=401` and `?page=9999` of the unfiltered view all served page 4. @@ -607,10 +614,10 @@ push for the same four reasons. | `0` | Complete (including a listing that ended before `--pages`: `stop_reason: listing_ended`) | | `1` | Crash | | `2` | Bad usage (including a URL that is not a supported Transfermarkt host) | -| `3` | Blocked — a bot-check/challenge page | +| `3` | Blocked — a bot-check/challenge page or a refusal. Never a server error: an HTTP 5xx is retried and then reported as `target_http_5xx` (exit 5 or 6) | | `4` | Zero rows, and the site's answer really was nothing: an empty squad, or the club, player or URL asked for does not exist (the site's own 404 page, or a profile id answered with a redirect elsewhere; `stop_reason: not_found`) | -| `5` | Nothing obtained. Either the content was never fetched or read — a load timeout, a dead exit, or a served page that shows rows the parser could not read (`stop_reason: parser_found_nothing`, a markup change) — or a 2Captcha product call failed on its own terms (the Fingerprint API, or a `--cdp-endpoint` connection: bad key, rate limit, a Scraping Browser profile already in use). Distinct from `3` and `4`: nothing can be concluded about the site from it. A captcha-SOLVE failure is deliberately NOT this — it's a warning and the run continues, reporting `3` only if the page really was blocked. | -| `6` | Partial — some pages fetched, then stopped early (including a later page that was refused, answered 404, parsed to nothing while showing rows, or served an earlier page again although the page before it linked onward: `stop_reason: page_repeated`). Not necessarily a fault: for `--mode market-values`, this is the expected result on the rare run whose pagination link needs the fallback chain (see "the gallery pagination trap") and runs out of it; on the common run it does not happen at all. | +| `5` | Nothing obtained. Either the content was never fetched or read — a load timeout, a dead exit, a server error from Transfermarkt that outlasted 2 retries (`stop_reason: target_http_5xx`), or a served page that shows rows the parser could not read (`stop_reason: parser_found_nothing`, a markup change) — or a 2Captcha product call failed on its own terms (the Fingerprint API, or a `--cdp-endpoint` connection: bad key, rate limit, a Scraping Browser profile already in use). Distinct from `3` and `4`: nothing can be concluded about the site from it. A captcha-SOLVE failure is deliberately NOT this — it's a warning and the run continues, reporting `3` only if the page really was blocked. | +| `6` | Partial — some pages fetched, then stopped early (including a later page that was refused, answered 404, parsed to nothing while showing rows, answered 5xx after 2 retries, or served an earlier page again although the page before it linked onward: `stop_reason: page_repeated`). Not necessarily a fault: for `--mode market-values`, this is the expected result on the rare run whose pagination link needs the fallback chain (see "the gallery pagination trap") and runs out of it; on the common run it does not happen at all. | **A run that finds nothing writes nothing.** Last known good output is not replaced with `[]`; `--allow-empty` is the opt-out. `.meta.json` diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 10a4915..7ac9c0a 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -50,8 +50,16 @@ as `no_new_products`, also complete. **`--mode transfers` stops after one page of about 20 rows.** On 2026-10-15 the default latest-transfers view (filtered by the site to players valued EUR 500,000+) held 20 transfers: one page, no pager. That is the site's whole -listing for that filter. The unfiltered listing is longer; see "How deep that -listing goes" in the README for its URL. +listing for that filter. `--min-value 0` reads the unfiltered listing (4 pages +that day); see "How deep that listing goes" in the README. + +## Exit 5 or 6 with `stop_reason: target_http_5xx` + +Transfermarkt answered with a server error (an HTTP 5xx, or the edge's +155-byte nginx `502 Bad Gateway` page) and still did after 2 more tries on +the same exit, `--retry-delay` and then twice that apart. Seen intermittently +on 2026-10-15, one page at a time. It is not a block and not an empty +result; run again later. ## Exit 6 with `stop_reason: page_repeated` diff --git a/page_flow.py b/page_flow.py index cccb41b..c10b97d 100644 --- a/page_flow.py +++ b/page_flow.py @@ -320,6 +320,12 @@ def wait_for_waf_challenge(content: Callable[[], Optional[str]], # all four engines reach the same answer whether or not they can see the # status code. "not_found": {"retry": False, "solve": False, "blocked": False}, + # The target's own 5xx: an HTTP 5xx status, or the edge's bare nginx 5xx + # page when no status is visible. A transient server error, not a + # refusal: never rotated past (`retry` here means "from another exit"), + # never paid for, never counted as blocked. Retried on the same exit by + # PageAttempts below, then the page is failed as STOP_TARGET_HTTP_5XX. + "server_error": {"retry": False, "solve": False, "blocked": False}, } @@ -340,6 +346,62 @@ def counts_as_blocked(state: str) -> bool: # does not exist, which is a fact about the site, not a failed fetch. STOP_NOT_FOUND = "not_found" +# Stop reason for a page the target answered with a 5xx after every retry. +# Not complete and not blocked: exit 5 on page 1, exit 6 on a later page. +STOP_TARGET_HTTP_5XX = "target_http_5xx" + +# How many times a 5xx page is fetched again on the same exit before it is +# failed. Measured 2026-10-15: the edge answered 3 of about 30 page loads in +# one evening with a 155-byte nginx 502, each time for a single page, and a +# later fetch of the same URL was served. Backoff: --retry-delay x 1, x 2. +SERVER_ERROR_RETRIES = 2 + + +class PageAttempts: + """The block-retry loop's iterator, plus a bounded same-exit retry for a + server error. + + Yields block-attempt indices 0..block_retries like `range()`. After + `again_after_server_error()` returns True, the NEXT step yields the same + index again, so a 5xx retry does not use up a proxy rotation and never + triggers one. The page's SolveBudget lives outside this loop and is not + reset by it. + """ + + def __init__(self, block_retries: int, retry_delay: float = 0.0, + server_retries: Optional[int] = None, + sleep: Optional[Callable[[float], None]] = None): + self.block_retries = block_retries + self.retry_delay = retry_delay + self.server_retries = (SERVER_ERROR_RETRIES if server_retries is None + else server_retries) + self.server_retries_used = 0 + self._sleep = sleep + self._index = 0 + self._repeat = False + + def __iter__(self): + while self._index <= self.block_retries: + self._repeat = False + yield self._index + if not self._repeat: + self._index += 1 + + def again_after_server_error(self, page_num: int, status=None) -> bool: + if self.server_retries_used >= self.server_retries: + return False + self.server_retries_used += 1 + pause = self.retry_delay * (2 ** (self.server_retries_used - 1)) + logger.warning("Page %d: the site answered with a server error (HTTP " + "%s) — fetching it again in %.1fs (%d/%d).", page_num, + status if status is not None else "5xx, read from the page", + pause, self.server_retries_used, self.server_retries) + if pause: + (self._sleep or __import__("time").sleep)(pause) + self._repeat = True + return True + + # Stop reason for a served page that holds rows the parser could not read: a # markup change, never an empty listing. Not complete, so exit 5 on page 1 # (nothing obtained) and exit 6 on a later page (rows kept, run incomplete). @@ -376,6 +438,8 @@ def failure_reason(outcome) -> str: return "page_load_timeout" if getattr(outcome, "parse_failed", False): return STOP_PARSER_FOUND_NOTHING + if getattr(outcome, "server_error", False): + return STOP_TARGET_HTTP_5XX if getattr(outcome, "state", None) == "not_found": # Only reachable on a page after the first: page 1's "not found" is # the run's answer (STOP_NOT_FOUND), a later page's is a page lost. @@ -467,6 +531,19 @@ def meta(self) -> dict: "listing_pages": self.repeat["page"] - 1} +def listing_filter_meta(mode: str, url: Optional[str]) -> dict: + """Sidecar keys naming the market-value floor a transfers run actually + used: the one in its URL (`--min-value`, or a typed --url), or the site's + own default view, whose floor is not in the URL (EUR 500,000 when + measured 2026-10-15; the site sets it, so it is not asserted here).""" + if mode != "transfers": + return {} + from product_parser import min_value_of + floor = min_value_of(url) + return {"min_market_value_eur": floor, + "market_value_filter": "url" if floor is not None else "site_default"} + + def settle_repeats(outcomes) -> PageSequence: """Run PageSequence over finished outcomes in page order — the concurrent path's version of the check the sequential loop makes as it goes. Pages diff --git a/playwright_scraper.py b/playwright_scraper.py index 1fe7c1a..ac42c74 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -87,7 +87,7 @@ next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, - MARKET_VALUES_URL, TRANSFERS_URL) + MARKET_VALUES_URL, TRANSFERS_URL, transfers_url) from output_writer import dedupe_by_key, finish_run, RemoteAPIError, EXIT_REMOTE_API_ERROR import page_flow from proxy_pool import (from_args as proxy_pool_from_args, to_playwright, mask, @@ -130,6 +130,8 @@ class PageOutcome: # decides whether a repeated page is the listing's end or a page # lost (page_flow.PageSequence). next_url: Optional[str] = None + # The target answered 5xx on every try (page_flow.STOP_TARGET_HTTP_5XX). + server_error: bool = False @property def ok(self) -> bool: @@ -137,7 +139,8 @@ def ok(self) -> bool: # page it is a page the run did not get. lost = self.state == "not_found" and self.page_num > 1 return (not self.load_failed and self.blocked_by is None - and not self.parse_failed and not lost) + and not self.parse_failed and not self.server_error + and not lost) def _driver(page): @@ -549,7 +552,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # to debug from. "Fail loudly" means saying WHAT failed. last_error, attempts_made = None, 0 - for block_attempt in range(block_retries + 1): + # A 5xx is fetched again on the same exit (bounded); a block moves on + # to the next exit. Both under the one budget above. + tries = page_flow.PageAttempts(block_retries, args.retry_delay) + for block_attempt in tries: logger.info("Fetching page %d: %s", page_num, url) load_failed, exit_failed = False, None resp_status, resp_headers = None, None @@ -626,6 +632,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome state = _classify(session.page, html, status=resp_status, headers=resp_headers) + if state == "server_error" and tries.again_after_server_error( + page_num, resp_status): + continue if not page_flow.should_retry(state): break @@ -658,6 +667,21 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.final_url = session.page.url return outcome + if state == "server_error": + # The target's own 5xx, still there after the same-exit retries: + # a page not served. Not a block (exit 3) and not an empty page. + debug_html = f"{args.out}_page{page_num}_debug.html" + with open(debug_html, "w", encoding="utf-8") as f: + f.write(html or "") + logger.error("Page %d: the site answered with a server error " + "after %d retr%s — saved to %s.", page_num, + tries.server_retries_used, + "y" if tries.server_retries_used == 1 else "ies", + debug_html) + outcome.server_error = True + outcome.final_url = session.page.url + return outcome + if state == "blocked": debug_html = f"{args.out}_page{page_num}_debug.html" with open(debug_html, "w", encoding="utf-8") as f: @@ -1042,7 +1066,8 @@ def scrape(args) -> int: pages_failed=failed_pages, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", start_url=args.url, final_url=final_url, - extra=pages.meta()) + extra={**pages.meta(), + **page_flow.listing_filter_meta(args.mode, args.url)}) def parse_args(): @@ -1070,6 +1095,12 @@ def parse_args(): help="Season start year for --mode club-squad, e.g. 2025 " "for 2025/26. Omit for the site's own current-season " "default.") + p.add_argument("--min-value", type=page_flow.bounded(int, 0, "--min-value"), + default=None, metavar="EUR", + help="Only --mode transfers: list transfers of players valued at least " + "this many EUR (0 = no floor). Without it the site's own default " + "view is read, which the site filters to EUR 500,000+ " + "(measured 2026-10-15).") p.add_argument("--player-id", default=None, help="Player id for --mode player, e.g. 418560 (Erling " "Haaland).") @@ -1163,6 +1194,11 @@ def parse_args(): p.error("--mode player needs --player-id (or --url)") args.url = player_url(args.player_id) + if args.min_value is not None: + if args.mode != "transfers": + p.error("--min-value applies to --mode transfers only") + args.url = transfers_url(args.min_value, base_url=args.url) + if not is_supported_host(args.url): why = unsupported_reason(args.url) p.error(f"{site_host(args.url) or args.url!r} {why}. Supported: " diff --git a/product_parser.py b/product_parser.py index d3ed33f..335dc2b 100644 --- a/product_parser.py +++ b/product_parser.py @@ -177,6 +177,50 @@ def is_supported_host(url: str) -> bool: MARKET_VALUES_URL = f"{BASE}/spieler-statistik/wertvollstespieler/marktwertetop" TRANSFERS_URL = f"{BASE}/statistik/neuestetransfers" +# The same listing with its filters in the URL. Measured 2026-10-15: the +# default view above applies the site's own floor of minMarktwert=500000 +# (players valued EUR 500,000+), carried by its filter form and tab links but +# not by its own URL; this path takes the floor, and the other filters it +# needs, as query parameters. `--min-value` builds it (see transfers_url). +TRANSFERS_FILTERED_PATH = "/transfers/neuestetransfers/statistik" +_TRANSFERS_FILTER_DEFAULTS = (("land_id", "0"), ("verein_land_id", "0"), + ("wettbewerb_id", "alle")) +_TRANSFERS_MAX_VALUE = "500000000" +MIN_VALUE_PARAM = "minMarktwert" + + +def transfers_url(min_value: int, base_url: Optional[str] = None) -> str: + """The latest-transfers listing filtered to players valued at least + `min_value` EUR (0 = unfiltered), on `base_url`'s host. + + The filters already in `base_url` (a league, a country) are kept; the + ones the filtered path needs and that are missing get the site's own + "all" values, as its unfiltered URL carries them. The page number is + dropped: pagination starts again from page 1.""" + from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode + parts = urlsplit(base_url or TRANSFERS_URL) + given = dict(parse_qsl(parts.query)) + given.pop(PAGE_PARAM, None) + query = [] + for key, default in _TRANSFERS_FILTER_DEFAULTS: + query.append((key, given.pop(key, default))) + given.pop(MIN_VALUE_PARAM, None) + query.append((MIN_VALUE_PARAM, str(int(min_value)))) + query.append(("maxMarktwert", given.pop("maxMarktwert", _TRANSFERS_MAX_VALUE))) + query.extend(given.items()) + return urlunsplit((parts.scheme or "https", parts.netloc or urlsplit(BASE).netloc, + TRANSFERS_FILTERED_PATH, urlencode(query), "")) + + +def min_value_of(url: Optional[str]) -> Optional[int]: + """The minMarktwert floor a listing URL carries, or None when it carries + none (the site's default view, whose floor is not in its URL).""" + from urllib.parse import urlsplit, parse_qsl + value = dict(parse_qsl(urlsplit(url or "").query)).get(MIN_VALUE_PARAM) + try: + return int(value) if value is not None else None + except ValueError: + return None def player_url(player_id: str, slug: str = "x") -> str: @@ -811,8 +855,8 @@ def is_not_found_page(html: Optional[str], status: Optional[int] = None) -> bool # cleared. In the second case the challenge's 202 had been discarded, as it # must be, so nothing but the body said the page had failed; it classified as # `empty`, and the run carried on and ended "complete" with page 1 missing. -# Read from the body, it classifies as the same failure the status already -# gave it where the status was visible. +# Read from the body, it classifies as the same `server_error` a visible 5xx +# status gives it. _GATEWAY_ERROR_RE = re.compile(r"\s*5\d\d [^<]{0,40}\s*\s*" r"\s*

\s*5\d\d", re.IGNORECASE) @@ -877,7 +921,7 @@ def page_states_rows(html: Optional[str], mode: str) -> bool: def detect_page_state(html: str, status: Optional[int] = None, url: Optional[str] = None, headers: Optional[Dict[str, str]] = None) -> str: - """content | blocked | captcha | empty | not_found — see + """content | blocked | captcha | empty | not_found | server_error — see page_flow.STATE_POLICY. The checks below are ordered by HOW MUCH EACH ONE PROVES, not by how @@ -937,14 +981,17 @@ def detect_page_state(html: str, status: Optional[int] = None, return "not_found" if is_gateway_error_page(html): - return "blocked" + return "server_error" vendor = detect_bot_challenge(html, url=url) if vendor: return "captcha" if vendor in SOLVABLE_VENDORS else "blocked" if not status_ok: - return "blocked" + # A 5xx with nothing in the body naming a vendor is the target's own + # server error, retried as such (page_flow "server_error"), never a + # refusal. Any other non-2xx stays a refusal. + return "server_error" if status >= 500 else "blocked" return "empty" diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 38e880d..5a631e5 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -59,7 +59,7 @@ next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, - MARKET_VALUES_URL, TRANSFERS_URL) + MARKET_VALUES_URL, TRANSFERS_URL, transfers_url) from output_writer import dedupe_by_key, finish_run, RemoteAPIError, EXIT_REMOTE_API_ERROR import page_flow from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, @@ -151,6 +151,8 @@ class PageOutcome: # decides whether a repeated page is the listing's end or a page # lost (page_flow.PageSequence). next_url: Optional[str] = None + # The target answered 5xx on every try (page_flow.STOP_TARGET_HTTP_5XX). + server_error: bool = False @property def ok(self) -> bool: @@ -158,7 +160,8 @@ def ok(self) -> bool: # page it is a page the run did not get. lost = self.state == "not_found" and self.page_num > 1 return (not self.load_failed and self.blocked_by is None - and not self.parse_failed and not lost) + and not self.parse_failed and not self.server_error + and not lost) class _Session: @@ -447,7 +450,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome budget = page_flow.SolveBudget() html, state, load_failed = None, "ok", False - for block_attempt in range(block_retries + 1): + # A 5xx is fetched again on the same exit (bounded); a block moves on + # to the next exit. Both under the one budget above. + tries = page_flow.PageAttempts(block_retries, args.retry_delay) + for block_attempt in tries: logger.info("Fetching page %d: %s", page_num, url) load_failed = False for attempt in range(1, args.retries + 1): @@ -487,6 +493,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome html = d["content"]() or "" state = page_flow.classify(html, url=page.url) + if state == "server_error" and tries.again_after_server_error( + page_num, None): + continue if not page_flow.should_retry(state): break @@ -519,6 +528,21 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.final_url = page.url return outcome + if state == "server_error": + # The target's own 5xx, still there after the same-exit retries: + # a page not served. Not a block (exit 3) and not an empty page. + debug_html = f"{args.out}_page{page_num}_debug.html" + with open(debug_html, "w", encoding="utf-8") as f: + f.write(html or "") + logger.error("Page %d: the site answered with a server error " + "after %d retr%s — saved to %s.", page_num, + tries.server_retries_used, + "y" if tries.server_retries_used == 1 else "ies", + debug_html) + outcome.server_error = True + outcome.final_url = d["current_url"]() + return outcome + if state == "blocked": debug_html = f"{args.out}_page{page_num}_debug.html" with open(debug_html, "w", encoding="utf-8") as f: @@ -722,7 +746,8 @@ def scrape(args) -> int: pages_failed=failed_pages, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", start_url=args.url, final_url=final_url, - extra=pages.meta()) + extra={**pages.meta(), + **page_flow.listing_filter_meta(args.mode, args.url)}) def parse_args(): @@ -732,6 +757,12 @@ def parse_args(): p.add_argument("--url", default=None) p.add_argument("--club-id", default=None) p.add_argument("--season", default=None) + p.add_argument("--min-value", type=page_flow.bounded(int, 0, "--min-value"), + default=None, metavar="EUR", + help="Only --mode transfers: list transfers of players valued at least " + "this many EUR (0 = no floor). Without it the site's own default " + "view is read, which the site filters to EUR 500,000+ " + "(measured 2026-10-15).") p.add_argument("--player-id", default=None) p.add_argument("--pages", type=page_flow.bounded(int, 1, "--pages"), default=1) p.add_argument("--delay", type=page_flow.bounded(float, 0, "--delay"), default=2.0) @@ -810,6 +841,11 @@ def parse_args(): p.error("--mode player needs --player-id (or --url)") args.url = player_url(args.player_id) + if args.min_value is not None: + if args.mode != "transfers": + p.error("--min-value applies to --mode transfers only") + args.url = transfers_url(args.min_value, base_url=args.url) + if not is_supported_host(args.url): why = unsupported_reason(args.url) p.error(f"{site_host(args.url) or args.url!r} {why}. Supported: " diff --git a/scraper_api_client.py b/scraper_api_client.py index 2e3884b..8a78a4a 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -135,7 +135,7 @@ page_states_rows, canonical_url, next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, - MARKET_VALUES_URL, TRANSFERS_URL) + MARKET_VALUES_URL, TRANSFERS_URL, transfers_url) from output_writer import finish_run, dedupe_by_key, EXIT_REMOTE_API_ERROR import page_flow import env_config @@ -325,7 +325,11 @@ def _fetch_one_page(args, page_num: int, url: str): read it off the live DOM.""" attempts = max(1, args.retries + 1) html, upstream_status, upstream_headers = "", None, {} - for attempt in range(1, attempts + 1): + # A 5xx from the target is fetched again on top of --retries (bounded, + # page_flow.SERVER_ERROR_RETRIES), so even --retries 0 rides out one. + tries = page_flow.PageAttempts(attempts - 1, args.retry_delay) + for attempt_index in tries: + attempt = attempt_index + 1 html, upstream_status, upstream_headers = fetch_html(args, url, args.timeout) if args.dump_html: @@ -338,6 +342,17 @@ def _fetch_one_page(args, page_num: int, url: str): state = detect_page_state(html, status=upstream_status, url=url, headers=upstream_headers) + if state == "server_error": + if tries.again_after_server_error(page_num, upstream_status): + continue + debug_html = f"{args.out}_page{page_num}_debug.html" + with open(debug_html, "w", encoding="utf-8") as f: + f.write(html) + logger.error("Page %d: the site answered with a server error " + "(upstream HTTP %s) after %d retries -- saved to %s. " + "A page not served, not a block.", page_num, + upstream_status, tries.server_retries_used, debug_html) + return [], None, upstream_status, html, page_flow.STOP_TARGET_HTTP_5XX if state == "blocked": # Retried like a vendor challenge, NOT returned immediately: a # transient upstream error (a non-2xx status, or an empty body) @@ -545,7 +560,8 @@ def scrape(args) -> int: pages_failed=pages_failed, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", start_url=args.url, final_url=final_url, - extra=pages.meta()) + extra={**pages.meta(), + **page_flow.listing_filter_meta(args.mode, args.url)}) def main() -> int: @@ -592,6 +608,12 @@ def parse_args(): "the mode's own canonical URL.") p.add_argument("--club-id", default=None, help="Club id for --mode club-squad.") p.add_argument("--season", default=None, help="Season start year for --mode club-squad.") + p.add_argument("--min-value", type=page_flow.bounded(int, 0, "--min-value"), + default=None, metavar="EUR", + help="Only --mode transfers: list transfers of players valued at least " + "this many EUR (0 = no floor). Without it the site's own default " + "view is read, which the site filters to EUR 500,000+ " + "(measured 2026-10-15).") p.add_argument("--player-id", default=None, help="Player id for --mode player.") p.add_argument("--pages", type=page_flow.bounded(int, 1, "--pages"), default=1, help="Listing pages to crawl. Checked per run against " @@ -662,6 +684,11 @@ def parse_args(): p.error("--mode player needs --player-id (or --url)") args.url = player_url(args.player_id) + if args.min_value is not None: + if args.mode != "transfers": + p.error("--min-value applies to --mode transfers only") + args.url = transfers_url(args.min_value, base_url=args.url) + if not is_supported_host(args.url): why = unsupported_reason(args.url) p.error(f"{site_host(args.url) or args.url!r} {why}. Supported: " diff --git a/selenium_scraper.py b/selenium_scraper.py index fa6e08e..37467cd 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -57,7 +57,7 @@ next_page_link, site_host, is_supported_host, unsupported_reason, HOSTS, player_url, club_squad_url, - MARKET_VALUES_URL, TRANSFERS_URL) + MARKET_VALUES_URL, TRANSFERS_URL, transfers_url) from output_writer import dedupe_by_key, finish_run, RemoteAPIError, EXIT_REMOTE_API_ERROR import page_flow from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, @@ -122,6 +122,8 @@ class PageOutcome: # decides whether a repeated page is the listing's end or a page # lost (page_flow.PageSequence). next_url: Optional[str] = None + # The target answered 5xx on every try (page_flow.STOP_TARGET_HTTP_5XX). + server_error: bool = False @property def ok(self) -> bool: @@ -129,7 +131,8 @@ def ok(self) -> bool: # page it is a page the run did not get. lost = self.state == "not_found" and self.page_num > 1 return (not self.load_failed and self.blocked_by is None - and not self.parse_failed and not lost) + and not self.parse_failed and not self.server_error + and not lost) class _Session: @@ -393,7 +396,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # ONE allowance for this page, across every block retry and rotation. budget = page_flow.SolveBudget() - for block_attempt in range(block_retries + 1): + # A 5xx is fetched again on the same exit (bounded); a block moves on + # to the next exit. Both under the one budget above. + tries = page_flow.PageAttempts(block_retries, args.retry_delay) + for block_attempt in tries: logger.info("Fetching page %d: %s", page_num, url) load_failed, exit_failed = False, None for attempt in range(1, args.retries + 1): @@ -434,6 +440,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome html = d["content"]() or "" state = page_flow.classify(html, url=d["current_url"]()) + if state == "server_error" and tries.again_after_server_error( + page_num, None): + continue if not page_flow.should_retry(state): break @@ -465,6 +474,21 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome outcome.final_url = d["current_url"]() return outcome + if state == "server_error": + # The target's own 5xx, still there after the same-exit retries: + # a page not served. Not a block (exit 3) and not an empty page. + debug_html = f"{args.out}_page{page_num}_debug.html" + with open(debug_html, "w", encoding="utf-8") as f: + f.write(html or "") + logger.error("Page %d: the site answered with a server error " + "after %d retr%s — saved to %s.", page_num, + tries.server_retries_used, + "y" if tries.server_retries_used == 1 else "ies", + debug_html) + outcome.server_error = True + outcome.final_url = d["current_url"]() + return outcome + if state == "blocked": debug_html = f"{args.out}_page{page_num}_debug.html" with open(debug_html, "w", encoding="utf-8") as f: @@ -665,7 +689,8 @@ def scrape(args) -> int: pages_failed=failed_pages, mode=args.mode, source=site_host(final_url) or "transfermarkt.com", start_url=args.url, final_url=final_url, - extra=pages.meta()) + extra={**pages.meta(), + **page_flow.listing_filter_meta(args.mode, args.url)}) def parse_args(): @@ -675,6 +700,12 @@ def parse_args(): p.add_argument("--url", default=None) p.add_argument("--club-id", default=None) p.add_argument("--season", default=None) + p.add_argument("--min-value", type=page_flow.bounded(int, 0, "--min-value"), + default=None, metavar="EUR", + help="Only --mode transfers: list transfers of players valued at least " + "this many EUR (0 = no floor). Without it the site's own default " + "view is read, which the site filters to EUR 500,000+ " + "(measured 2026-10-15).") p.add_argument("--player-id", default=None) p.add_argument("--pages", type=page_flow.bounded(int, 1, "--pages"), default=1) p.add_argument("--delay", type=page_flow.bounded(float, 0, "--delay"), default=2.0) @@ -736,6 +767,11 @@ def parse_args(): p.error("--mode player needs --player-id (or --url)") args.url = player_url(args.player_id) + if args.min_value is not None: + if args.mode != "transfers": + p.error("--min-value applies to --mode transfers only") + args.url = transfers_url(args.min_value, base_url=args.url) + if not is_supported_host(args.url): why = unsupported_reason(args.url) p.error(f"{site_host(args.url) or args.url!r} {why}. Supported: " diff --git a/smoke_test.py b/smoke_test.py index e25e61a..91f54b3 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -4306,15 +4306,16 @@ def test_csv_formula_escape(): BOUNDED_FLAGS = { "playwright_scraper.py": {"--pages": 1, "--delay": 0, "--concurrency": 1, "--retry-delay": 0, "--proxy-sessions": 1, - "--proxy-block-retries": 0}, + "--proxy-block-retries": 0, "--min-value": 0}, "puppeteer_scraper.py": {"--pages": 1, "--delay": 0, "--concurrency": 1, "--retry-delay": 0, "--proxy-sessions": 1, - "--proxy-block-retries": 0}, + "--proxy-block-retries": 0, "--min-value": 0}, "selenium_scraper.py": {"--pages": 1, "--delay": 0, "--concurrency": 1, "--retry-delay": 0, "--proxy-sessions": 1, - "--proxy-block-retries": 0}, + "--proxy-block-retries": 0, "--min-value": 0}, "scraper_api_client.py": {"--pages": 1, "--delay": 0, "--timeout": 1, - "--retries": 0, "--retry-delay": 0}, + "--retries": 0, "--retry-delay": 0, + "--min-value": 0}, } # Numeric, and deliberately not a count: a score threshold. UNBOUNDED_NUMERIC_FLAGS = {"--min-score"} @@ -4752,7 +4753,7 @@ def patch(name, value): os.environ["TWOCAPTCHA_KEY"] = "test-placeholder" sys.argv = [modname + ".py", "--mode", "transfers", "--url", url, "--pages", str(pages), "--delay", "0", "--out", out, - "--format", "json"] + list(extra_argv) + "--format", "json", "--retry-delay", "0"] + list(extra_argv) args = mod.parse_args() if modname == "scraper_api_client": patch("fetch_html", lambda a, u, timeout: @@ -4869,10 +4870,10 @@ def test_listing_end_and_repeated_pages(skips): # A bare nginx 5xx is a page not served, with or without its status. from product_parser import detect_page_state, is_gateway_error_page - ok &= check("the edge's 502 page, with no status, is not an empty page", - detect_page_state(FIX_NGINX_502) == "blocked") + ok &= check("the edge's 502 page, with no status, is a server error, not an empty page", + detect_page_state(FIX_NGINX_502) == "server_error") ok &= check("...and classifies the same with its status", - detect_page_state(FIX_NGINX_502, status=502) == "blocked") + detect_page_state(FIX_NGINX_502, status=502) == "server_error") ok &= check("the 502 marker matches none of the served or 404 captures", not any(is_gateway_error_page(h) for h in ( FIX_TRANSFERS_ONE_PAGE_P1, FIX_TRANSFERS_PAGED_P1, @@ -4882,9 +4883,8 @@ def test_listing_end_and_repeated_pages(skips): gateway_first = lambda n: FIX_NGINX_502 if n == 1 else FIX_TRANSFERS_PAGED_P2 rc, meta, skus = _run_listing("scraper_api_client", TRANSFERS_PAGED_URL, 2, gateway_first) - ok &= check("a 502 for page 1 is not a complete run (got rc=%s %s)" - % (rc, meta.get("status")), - rc != 0 and meta.get("status") != "complete") + ok &= check("a 502 for page 1 is exit 5, not blocked and not complete (got rc=%s)" + % rc, rc == 5 and meta.get("status") != "complete") # The runs below replace each engine's _fetch_one_page, so whether the # real one records the page's next link is checked on its source. @@ -4945,6 +4945,332 @@ def test_listing_end_and_repeated_pages(skips): return ok +class _Site: + """Answers each URL from a per-page script of (status, html) replies; the + last reply repeats. Counts the requests per page.""" + + def __init__(self, script): + self.script = script # {page_num: [(status, html), ...]} + self.hits = {} + + def serve(self, url): + from product_parser import page_number_from_url + n = page_number_from_url(url) or 1 + replies = self.script[n] + i = self.hits.get(n, 0) + self.hits[n] = i + 1 + return replies[min(i, len(replies) - 1)] + + +def _select(html, selector): + from bs4 import BeautifulSoup + return BeautifulSoup(html or "", "html.parser").select(selector) + + +class _PWPage: + def __init__(self, site): + self.site, self.url, self.html = site, "about:blank", "" + + def goto(self, url, **_kw): + status, self.html = self.site.serve(url) + self.url = url + return type("Resp", (), {"status": status, "headers": {}})() + + def content(self): + return self.html + + def query_selector_all(self, selector): + return _select(self.html, selector) + + def query_selector(self, selector): + found = _select(self.html, selector) + if not found: + return None + href = found[0].get("href") + return type("El", (), {"get_attribute": lambda self, _n: href})() + + def evaluate(self, _js): + return None + + def wait_for_timeout(self, _ms): + pass + + def screenshot(self, **_kw): + pass + + +class _PupPage(_PWPage): + async def goto(self, url, *_a, **_kw): + return _PWPage.goto(self, url) + + async def content(self): + return self.html + + async def querySelectorAll(self, selector): + return _select(self.html, selector) + + async def evaluate(self, *_a, **_kw): + return None + + async def screenshot(self, *_a, **_kw): + return None + + +class _SelDriver: + def __init__(self, site): + self.site, self.current_url, self.page_source = site, "about:blank", "" + + def get(self, url): + _status, self.page_source = self.site.serve(url) + self.current_url = url + + def find_elements(self, _by, selector): + return _select(self.page_source, selector) + + def execute_script(self, *_a): + return None + + def save_screenshot(self, *_a): + return True + + +class _Bridge: + def run(self, coro, timeout=None): + import asyncio + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + def close(self): + pass + + +def _run_site(modname, url, pages, site, extra_argv=()): + """One entry point's real scrape() and real _fetch_one_page against a + scripted site; only the browser (or the Scraper API call) is replaced. + Returns (rc, meta).""" + mod = importlib.import_module(modname) + tmp = tempfile.mkdtemp(prefix="tm_site_") + out = os.path.join(tmp, "out") + saved, saved_argv = {}, sys.argv + env_names = set(env_config.ENV_KEYS) | {"TWOCAPTCHA_KEY"} + saved_env = {k: os.environ.get(k) for k in env_names} + + def patch(name, value): + saved[name] = getattr(mod, name) + setattr(mod, name, value) + + class Session: + pool = None + + def open(self): + return self + + def close(self): + pass + + def relaunch(self): + return self + + session = Session() + try: + for k in env_names: + os.environ.pop(k, None) + if modname == "scraper_api_client": + os.environ["TWOCAPTCHA_KEY"] = "test-placeholder" + sys.argv = [modname + ".py", "--mode", "transfers", "--url", url, + "--pages", str(pages), "--delay", "0", "--retry-delay", "0", + "--out", out, "--format", "json", "--allow-empty"] + list(extra_argv) + args = mod.parse_args() + if modname == "scraper_api_client": + def fetch(a, u, timeout): + status, html = site.serve(u) + return html, status, {} + patch("fetch_html", fetch) + elif modname == "playwright_scraper": + session.page = _PWPage(site) + patch("sync_playwright", lambda: _FakePlaywright()) + patch("_BrowserSession", lambda pw, a, pool, **kw: session) + elif modname == "puppeteer_scraper": + session.page, session.bridge = _PupPage(site), _Bridge() + patch("_AsyncBridge", lambda: session.bridge) + patch("_Session", lambda *a, **kw: session) + else: + session.driver = _SelDriver(site) + patch("_Session", lambda *a, **kw: session) + rc = mod.scrape(args) + finally: + for name, value in saved.items(): + setattr(mod, name, value) + sys.argv = saved_argv + for k, v in saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + meta = {} + if os.path.exists(out + ".meta.json"): + meta = json.load(open(out + ".meta.json", encoding="utf-8")) + shutil.rmtree(tmp, ignore_errors=True) + return rc, meta + + +def test_target_5xx_is_a_server_error(skips): + group("a 5xx from the site is a server error: retried, then failed, never blocked") + from product_parser import detect_page_state + ok = True + ok &= check("HTTP 500 with an ordinary error body is a server error", + detect_page_state("Internal error", + status=500) == "server_error") + ok &= check("a 5xx that carries a vendor challenge is still not a server error", + detect_page_state(FIX_AWS_WAF_CHALLENGE, status=503) != "server_error") + ok &= check("a 403 stays a refusal", + detect_page_state("Forbidden", + status=403) == "blocked") + ok &= check("server_error is neither blocked, rotated past nor paid for", + not page_flow.counts_as_blocked("server_error") + and not page_flow.should_retry("server_error") + and not page_flow.should_solve("server_error")) + + slept = [] + tries = page_flow.PageAttempts(0, 1.5, sleep=slept.append) + seen = [] + for i in tries: + seen.append(i) + if not tries.again_after_server_error(1, 502): + break + ok &= check("PageAttempts: 2 same-exit retries, then stops (yielded %s)" % seen, + seen == [0, 0, 0] and tries.server_retries_used == 2) + ok &= check("PageAttempts: backoff is --retry-delay x1, x2 (%s)" % slept, + slept == [1.5, 3.0]) + tries = page_flow.PageAttempts(2, 0) + ok &= check("PageAttempts: without a retry it is range(block_retries + 1)", + list(tries) == [0, 1, 2]) + + e502 = (502, FIX_NGINX_502) + p1, p2 = (200, FIX_TRANSFERS_PAGED_P1), (200, FIX_TRANSFERS_PAGED_P2) + for modname in ("scraper_api_client", "playwright_scraper", + "puppeteer_scraper", "selenium_scraper"): + try: + importlib.import_module(modname) + except ImportError as e: + skips.append("%s 5xx run (%s)" % (modname, e)) + continue + site = _Site({1: [e502, e502, p1], 2: [p2]}) + rc, meta = _run_site(modname, TRANSFERS_PAGED_URL, 2, site) + ok &= check("%s: two 502s, then served -> exit 0, page 1 fetched 3 times " + "(got rc=%s hits=%s)" % (modname, rc, site.hits), + rc == 0 and site.hits.get(1) == 3 + and meta.get("pages_completed") == 2 and meta.get("products") == 6) + site = _Site({1: [e502]}) + rc, meta = _run_site(modname, TRANSFERS_PAGED_URL, 2, site) + ok &= check("%s: page 1 a 502 every time -> exit 5, target_http_5xx, " + "3 fetches, never exit 3 (got rc=%s %s hits=%s)" + % (modname, rc, meta.get("stop_reason"), site.hits), + rc == 5 and meta.get("stop_reason") == "target_http_5xx" + and site.hits.get(1) == 3 and meta.get("status") != "complete") + site = _Site({1: [p1], 2: [e502]}) + rc, meta = _run_site(modname, TRANSFERS_PAGED_URL, 2, site) + ok &= check("%s: page 2 a 502 every time -> exit 6, target_http_5xx, " + "page 2 failed (got rc=%s %s %s)" + % (modname, rc, meta.get("stop_reason"), meta.get("pages_failed")), + rc == 6 and meta.get("stop_reason") == "target_http_5xx" + and meta.get("pages_failed") == [2] and site.hits.get(2) == 3) + return ok + + +def test_min_value_flag(skips): + group("--min-value: the latest-transfers market-value floor") + from product_parser import transfers_url, min_value_of + ok = True + ok &= check("--min-value 0 builds the unfiltered listing URL measured 2026-10-15", + transfers_url(0) == TRANSFERS_PAGED_URL) + kept = transfers_url(250000, base_url=TRANSFERS_PAGED_URL.replace( + "wettbewerb_id=alle", "wettbewerb_id=GB1") + "&page=3") + ok &= check("a typed URL's own filters are kept, its page dropped (%s)" % kept, + "wettbewerb_id=GB1" in kept and "minMarktwert=250000" in kept + and "page=" not in kept) + ok &= check("min_value_of reads the floor, None for the default view", + min_value_of(TRANSFERS_PAGED_URL) == 0 + and min_value_of(TRANSFERS_URL) is None) + ok &= check("the sidecar names the floor, or the site's default view", + page_flow.listing_filter_meta("transfers", TRANSFERS_PAGED_URL) + == {"min_market_value_eur": 0, "market_value_filter": "url"} + and page_flow.listing_filter_meta("transfers", TRANSFERS_URL) + == {"min_market_value_eur": None, "market_value_filter": "site_default"} + and page_flow.listing_filter_meta("market-values", MARKET_VALUES_URL) == {}) + + saved_argv = sys.argv + env_names = set(env_config.ENV_KEYS) | {"TWOCAPTCHA_KEY"} + saved_env = {k: os.environ.get(k) for k in env_names} + try: + for k in env_names: + os.environ.pop(k, None) + os.environ["TWOCAPTCHA_KEY"] = "test-placeholder" + for modname in ("scraper_api_client", "playwright_scraper", + "puppeteer_scraper", "selenium_scraper"): + try: + mod = importlib.import_module(modname) + except ImportError as e: + skips.append("%s --min-value parse_args (%s)" % (modname, e)) + continue + for argv, want in ((["--mode", "transfers"], TRANSFERS_URL), + (["--mode", "transfers", "--min-value", "0"], + TRANSFERS_PAGED_URL), + (["--mode", "transfers", "--min-value", "500000"], + TRANSFERS_PAGED_URL.replace("minMarktwert=0", + "minMarktwert=500000"))): + sys.argv = [modname + ".py"] + argv + ok &= check("%s %s -> %s" % (modname, " ".join(argv), want), + mod.parse_args().url == want) + for argv in (["--mode", "transfers", "--min-value", "-1"], + ["--mode", "transfers", "--min-value", "x"], + ["--mode", "market-values", "--min-value", "0"]): + sys.argv = [modname + ".py"] + argv + code = None + try: + with open(os.devnull, "w") as devnull: + saved_err, sys.stderr = sys.stderr, devnull + try: + mod.parse_args() + finally: + sys.stderr = saved_err + except SystemExit as e: + code = e.code + ok &= check("%s %s -> exit 2" % (modname, " ".join(argv)), code == 2) + finally: + sys.argv = saved_argv + for k, v in saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + for modname in ("scraper_api_client", "selenium_scraper"): + try: + importlib.import_module(modname) + except ImportError as e: + skips.append("%s --min-value run (%s)" % (modname, e)) + continue + paged = {1: FIX_TRANSFERS_PAGED_P1, 2: FIX_TRANSFERS_PAGED_P2} + rc, meta, skus = _run_listing(modname, TRANSFERS_URL, 2, + lambda n: paged[n], ("--min-value", "0")) + ok &= check("%s --min-value 0: the sidecar records min_market_value_eur 0 " + "and the run reads the filtered path (got %s)" + % (modname, meta.get("start_url")), + rc == 0 and meta.get("min_market_value_eur") == 0 + and meta.get("market_value_filter") == "url" + and meta.get("start_url") == TRANSFERS_PAGED_URL and len(skus) == 6) + rc, meta, skus = _run_listing(modname, TRANSFERS_URL, 1, + lambda n: FIX_TRANSFERS_ONE_PAGE_P1) + ok &= check("%s without --min-value: the sidecar says site_default" % modname, + rc == 0 and meta.get("min_market_value_eur") is None + and meta.get("market_value_filter") == "site_default") + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -5008,6 +5334,8 @@ def main() -> int: ok &= test_credential_scan_judges_paths_inside_the_repo() ok &= test_no_private_citations() ok &= test_listing_end_and_repeated_pages(skips) + ok &= test_target_5xx_is_a_server_error(skips) + ok &= test_min_value_flag(skips) print() if _failures: