diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cba3812 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git/ +.github/ +.env +*.log +*.key +secrets* +__pycache__/ +*.py[cod] +.pytest_cache/ +.fingerprint_cache/ +.venv/ +venv/ +env/ +amazon_products.* +*_debug.html +*_debug.png +live_results/ +live_results.zip diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d7b23ce --- /dev/null +++ b/.env.example @@ -0,0 +1,86 @@ +# amazon-scraper — configuration +# +# Copy this file to `.env` next to the scripts and fill in what you need. +# Nothing here is required: with no .env and no environment variables at all +# the scrapers behave exactly as their flags say. +# +# Measured, so you know what the paid parts actually buy: an ordinary local +# Chromium on a clean residential IP returned 3 pages / 77 products from +# amazon.com with NO key, NO proxy and NO account — see the README. The +# products below buy volume from many addresses, a specific exit country, and +# a browser you do not have to run yourself. +# +# CREDENTIALS BELONG HERE AND NOWHERE ELSE. A secret on a command line is +# readable by anything on the machine that can run `ps`, and it lands in your +# shell history. Every flag these variables back can also be typed, and an +# explicit flag always wins over this file. +# +# Precedence, highest first: +# explicit flag -> exported environment variable -> .env -> default +# +# Run `python3 env_config.py` to see what was picked up. It prints which +# variables were found WITHOUT printing their values. + +# --------------------------------------------------------------------------- +# 2Captcha — one key, four separately-billed products +# --------------------------------------------------------------------------- +# Captcha solving, the Scraping Browser API, proxies and fingerprints all +# authenticate with this one key and are billed separately. +# +# On Amazon the solving part is for ONE specific page: the first-party image +# captcha ("Enter the characters you see below"). It is NOT needed for the +# AWS WAF interstitial, which the browser clears by itself in about four +# seconds, and no solve helps a 503 throttle or a sign-in wall. +TWOCAPTCHA_KEY=your_2captcha_api_key_here + +# --------------------------------------------------------------------------- +# Scraping Browser API (a remote browser over CDP) +# --------------------------------------------------------------------------- +# Endpoint shape: +# ws://{login}-zone-scraping_browser-country-{cc}-pid-{profileId}:{password}@cb.2captcha.com:9222 +# +# `country-` picks the exit country. On Amazon that decides the page language +# AND the currency you are quoted — prices are converted for a visitor whose +# delivery country differs from the marketplace, so a European exit reading +# amazon.co.jp is shown "EUR 25.34" rather than yen. If you want a +# marketplace's own currency, use an exit in that country. +# +# `pid-` is a profile with persistent cookies and allows ONE live connection +# each, capped per account — reuse profile ids rather than minting one per +# run. Note the cookie persistence has a cost here: Amazon assigns some page +# variants per session, and a profile that was given the detail page without +# its reviews keeps getting it. A different `pid` is the fix. +# +# --concurrency is refused with this set, for the same one-connection reason. +AMAZON_CDP_ENDPOINT= + +# --------------------------------------------------------------------------- +# Proxy +# --------------------------------------------------------------------------- +# One exit, as a full URL: http://ACCOUNT:PASSWORD@HOST:PORT +# For a pool, use --proxy-file with one URL per line. +# +# Ignored when AMAZON_CDP_ENDPOINT is set: the remote browser has its own +# exit, and layering a second on top contradicts it rather than helping. +# +# Note selenium_scraper.py cannot authenticate a proxy at all — chromedriver's +# --proxy-server takes no credentials. It strips them and warns. Use the +# Playwright or pyppeteer engine for an authenticated exit. +AMAZON_PROXY= + +# --------------------------------------------------------------------------- +# Target +# --------------------------------------------------------------------------- +# The URL to scrape, so a scheduled run does not need it on the command line. +# The hostname decides which of the 21 marketplaces is read — there is no +# separate marketplace flag, so a flag and a URL cannot disagree. +# +# listing mode: https://www.amazon.com/s?k=wireless+headphones +# https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics/ +# product mode: https://www.amazon.com/dp/B07K5214NZ +# reviews mode: https://www.amazon.com/dp/B07K5214NZ +AMAZON_URL= + +# Deliberately NOT configurable here: the output prefix. `--out` already has a +# non-empty default, so the loader (which only fills UNSET values) would never +# apply a variable for it — a setting that looks configurable and is not. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..2d75e54 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,116 @@ +name: Bug report +description: The scraper ran but the result was wrong, empty, or it crashed. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + **Before anything else: never paste an API key, a proxy password or a + full `ws://user:pass@...` endpoint into a public issue.** The scrapers + mask credentials in their own logs, but raw HTML dumps and the Scraper + API's `x-debug` header do not. Replace them with `***`. + + Two things answer most reports faster than we can: + + - **Zero products?** Check you used a *filtered* category URL, not the + bare `/shopping/kids/items.aspx` hub — the hub carries no product + JSON-LD and correctly returns nothing. See + [TROUBLESHOOTING.md](../blob/main/TROUBLESHOOTING.md). + - **Wrong currency or localised titles?** Amazon decides both from + your exit IP, not from the URL. See "Geo-redirect" in the README. + + - type: dropdown + id: engine + attributes: + label: Which engine + options: + - playwright_scraper.py + - puppeteer_scraper.py + - selenium_scraper.py + - scraper_api_client.py + - fingerprint_client.py + - other / not sure + validations: + required: true + + - type: dropdown + id: browser_route + attributes: + label: How was the browser reached + options: + - A browser the script launched itself + - A remote browser over --cdp-endpoint + - No browser (scraper_api_client.py over plain HTTP) + - Not applicable + validations: + required: true + + - type: input + id: exit_code + attributes: + label: Exit code + description: >- + `echo $?` straight after the run. These mean specific things — 3 is a + challenge page, 4 is "ran fine, found nothing", 124 is a self-imposed + timeout. The table is in the README. + placeholder: "4" + validations: + required: true + + - type: textarea + id: command + attributes: + label: The exact command + description: With credentials replaced by `***`. + render: shell + placeholder: | + python3 playwright_scraper.py \ + --url "https://www.amazon.com/s?k=wireless+headphones" \ + --pages 1 --out girls + validations: + required: true + + - type: textarea + id: output + attributes: + label: Output + description: >- + The whole run, not just the last line — the useful detail is usually + upstream of the error. Credentials replaced by `***`. + render: text + validations: + required: true + + - type: textarea + id: expected + attributes: + label: What you expected instead + placeholder: >- + 96 products, as the README says a category page yields. + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: | + Please run and paste: + ``` + python3 -c "import sys, platform; print(sys.version, platform.platform())" + pip list 2>/dev/null | grep -Ei "playwright|pyppeteer|selenium|beautifulsoup4|requests|webdriver" + python3 smoke_test.py | tail -3 + ``` + render: text + validations: + required: true + + - type: checkboxes + id: checks + attributes: + label: Checks + options: + - label: The offline suite passes on my machine (`python3 smoke_test.py`) + required: true + - label: I removed every API key, password and credentialled URL from this report + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..21e7321 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: true +contact_links: + - name: 2Captcha support + url: https://2captcha.com/support + about: >- + Billing, an API key that will not authenticate, profile or quota limits + (ERROR_MAX_PROFILES), proxy availability. Anything about the service + rather than about this code — support can see your account, we cannot. + - name: 2Captcha API documentation + url: https://2captcha.com/api-docs + about: Task types, parameters and error codes for the solver API. + - name: Other scrapers in this family + url: https://github.com/2scraper + about: Same structure, other sites. diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 0000000..db40153 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,30 @@ +name: Question +description: How do I ... ? +labels: ["question"] +body: + - type: markdown + attributes: + value: | + Worth a look first, in this order: + + - [TROUBLESHOOTING.md](../blob/main/TROUBLESHOOTING.md) — ordered by how + often each thing actually happens + - The README section on the 2Captcha product you are using + - `python3 env_config.py` — reports what credentials are configured, + without printing them + + - type: textarea + id: question + attributes: + label: What are you trying to do + description: >- + The goal, not only the step that is stuck — often there is a shorter + route to it. + validations: + required: true + + - type: textarea + id: tried + attributes: + label: What you tried + render: shell diff --git a/.github/ISSUE_TEMPLATE/site_changed.yml b/.github/ISSUE_TEMPLATE/site_changed.yml new file mode 100644 index 0000000..e4d4d96 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/site_changed.yml @@ -0,0 +1,58 @@ +name: Site changed / selectors broke +description: Amazon changed something and the parser no longer matches. +labels: ["site-change"] +body: + - type: markdown + attributes: + value: | + This is the most common way a scraper stops working, and it is worth its + own report because the fix is different from a code bug: something on + amazon.com moved. + + Useful to know before filing: the parser tries **JSON-LD first**, then a + CSS + URL-pattern fallback. Which one broke narrows the fix a lot. + + - type: dropdown + id: what_broke + attributes: + label: What stopped matching + options: + - Products (0 rows on a page that visibly has products) + - A specific field is null or wrong (price, brand, sku, image, URL) + - Pagination stops early or repeats a page + - The captcha detector finds nothing on a page that has one + - Something else + validations: + required: true + + - type: input + id: url + attributes: + label: URL + description: The exact listing URL, and the country your exit IP was in. + placeholder: "https://www.amazon.com/s?k=wireless+headphones (marketplace: .com, exit IP: DE)" + validations: + required: true + + - type: textarea + id: evidence + attributes: + label: Evidence from the page + description: | + Whichever of these you can get. A capture beats a description. + + - JSON-LD as the page actually serves it: + `python3 -c "import json,sys,re;h=open('dump.html').read();print([m[:400] for m in re.findall(r'',h,re.S)][:2])"` + - Or open devtools and paste one `Product` node from the `ItemList`. + - For a field problem: the value you got, and the value on the page. + render: text + validations: + required: true + + - type: textarea + id: notes + attributes: + label: Anything else + description: >- + Whether it reproduces on a second engine, whether it reproduces from a + different country, and when it last worked. diff --git a/.github/ci_checks.py b/.github/ci_checks.py new file mode 100644 index 0000000..fccb77e --- /dev/null +++ b/.github/ci_checks.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +CI checks that are too long to live inside the workflow YAML. + +They started out as heredocs in `.github/workflows/tests.yml` and moved here +for one practical reason: a Python block nested inside YAML inside a shell +`run:` needs three levels of quoting to stay intact, and shell-quoted regexes +like '(ws|wss)://[^ "'"'"']+' do not survive being copied through a browser. +A separate .py file is copy-paste safe, runs locally, and can be read on its +own. + +Run any of these from the repo root: + + python .github/ci_checks.py --help-check + python .github/ci_checks.py --sample-check + python .github/ci_checks.py --secret-check + python .github/ci_checks.py --all + +Each prints what it looked at and exits non-zero on failure. +""" + +import argparse +import csv +import json +import re +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent + +# Engine libraries are deliberately absent in CI — the offline suite does not +# need them. An ImportError naming one of these is expected, not a failure. +ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager") + +CLIS = ["playwright_scraper.py", "puppeteer_scraper.py", "selenium_scraper.py", + "scraper_api_client.py", "fingerprint_client.py", "env_config.py"] + +SAMPLE_FILES = ("sample_output.json", "sample_output.csv") + +# Phrases that show up in hand-written or template sample data. The point of +# committing a sample is that it came from a real run; a placeholder teaches +# readers field names and value shapes that do not exist. +FABRICATION_MARKERS = ("sample-product-", "example brand", "sample product", + "product description text", "lorem ipsum", + "your_api_key", "123456789") + +# A URL carrying real credentials — the shape is scheme://something:something@host +# (deliberately not spelled out as an example here: this file scans itself, and +# an illustrative credential in a comment is a false positive that turns the +# build red for no reason. It happened on the first run.) +CREDENTIALLED_URL = re.compile(r"(?:ws|wss|https?)://[^\s\"'/]+:[^\s\"'/]+@") + +# Documented placeholders, which are supposed to look like the real thing. +CREDENTIAL_ALLOWED = ("USER:PASS", "user:pass", "ACCOUNT:PASSWORD", "{login}", + "***", "password}@", "u:p@h", "LOGIN:PASSWORD") + +# A 2captcha API key is a 32-character hex string. +HEX32 = re.compile(r"\b[0-9a-f]{32}\b") +HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest", "checksum") + +SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example") + + +def scanned_files(): + for path in sorted(REPO.rglob("*")): + if not path.is_file() or path.suffix not in SCANNED_SUFFIXES: + continue + if any(part in {".git", "__pycache__", ".venv", "venv"} + for part in path.parts): + continue + yield path + + +def help_check(): + failed = [] + for name in CLIS: + script = REPO / name + if not script.is_file(): + print(f"missing {name}") + failed.append(name) + continue + result = subprocess.run([sys.executable, str(script), "--help"], + capture_output=True, text=True, cwd=REPO) + if result.returncode == 0: + print(f"ok {name}") + continue + blob = result.stdout + result.stderr + if "ModuleNotFoundError" in blob and any(lib in blob for lib in ENGINE_LIBS): + print(f"skipped {name} (engine library not installed here)") + continue + print(f"FAILED {name}\n{blob}") + failed.append(name) + return failed + + +def sample_check(): + failed = [] + for name in SAMPLE_FILES: + if not (REPO / name).is_file(): + failed.append(f"{name} is missing — regenerate it from a real run") + if failed: + return failed + + rows = json.loads((REPO / "sample_output.json").read_text(encoding="utf-8")) + if not rows: + return ["sample_output.json is empty — a run that found nothing is not a sample"] + + blob = json.dumps(rows).lower() + hits = [m for m in FABRICATION_MARKERS if m in blob] + if hits: + failed.append(f"sample_output.json looks fabricated: {hits}") + + # The committed sample doubles as a schema test: rename a field in the code + # and forget the sample, and this fails rather than the docs going stale. + sys.path.insert(0, str(REPO)) + from dataclasses import asdict + + from output_writer import Product + expected = list(asdict(Product()).keys()) + + for i, row in enumerate(rows): + if list(row.keys()) != expected: + failed.append(f"sample_output.json row {i}: columns differ from " + f"output_writer.Product") + break + + with (REPO / "sample_output.csv").open(newline="", encoding="utf-8") as handle: + header = next(csv.reader(handle)) + if header != expected: + failed.append("sample_output.csv header differs from output_writer.Product") + + if not failed: + discounted = sum(1 for r in rows if r.get("original_price")) + print(f"ok {len(rows)} rows, {len(expected)} columns, " + f"{discounted} discounted, schema matches") + return failed + + +def secret_check(): + failed = [] + scanned = 0 + for path in scanned_files(): + scanned += 1 + for lineno, line in enumerate( + path.read_text(encoding="utf-8", errors="replace").splitlines(), 1): + rel = path.relative_to(REPO) + + if CREDENTIALLED_URL.search(line) and not any( + token in line for token in CREDENTIAL_ALLOWED): + failed.append(f"{rel}:{lineno} looks like a URL with real " + f"credentials in it") + + for match in HEX32.findall(line): + if any(token in line.lower() for token in HEX32_ALLOWED): + continue + failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " + f"32-char hex string, the shape of a 2captcha key") + + if not failed: + print(f"ok {scanned} files scanned, nothing credential-shaped") + return failed + + +CHECKS = {"help": help_check, "sample": sample_check, "secret": secret_check} + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--help-check", action="store_true", + help="Every shipped CLI answers --help") + parser.add_argument("--sample-check", action="store_true", + help="sample_output.* exist, are real, match the schema") + parser.add_argument("--secret-check", action="store_true", + help="No credentials committed anywhere") + parser.add_argument("--all", action="store_true", help="All of the above") + args = parser.parse_args() + + selected = [name for name in CHECKS + if args.all or getattr(args, f"{name}_check")] + if not selected: + parser.error("pick at least one check, or --all") + + failures = [] + for name in selected: + print(f"--- {name} check") + failures += [f"[{name}] {line}" for line in CHECKS[name]()] + + if failures: + print() + for line in failures: + print("FAILED:", line) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml new file mode 100644 index 0000000..34541b0 --- /dev/null +++ b/.github/workflows/canary.yml @@ -0,0 +1,195 @@ +name: canary + +# tests.yml (deliberately) never touches amazon.com — see its own header for +# why. That leaves a real gap: a markup change on the site itself only shows up +# when a user hits it and files an issue, while the offline badge stays green +# throughout. This workflow is the other half — a real run, on a real search +# URL, on a schedule — so a site-side break is caught here first. +# +# Uses no 2Captcha product and no secret by default, because the README's +# headline claim is that an ordinary browser on a clean IP returns three pages +# of products with no key, no proxy and no account, and this is where that +# claim gets exercised continuously. +# +# EXPECT MORE NOISE HERE THAN ON A RESIDENTIAL IP. GitHub Actions runners are +# datacenter addresses, and Amazon treats those less generously than the +# residential exit the README's numbers were measured from: a red run can mean +# the site changed (a real break) or that this address was throttled or shown a +# captcha. The exit-code interpretation step below spells out which is which, +# and if it proves too noisy, set the AMAZON_PROXY repo secret and uncomment +# the --proxy line to run from a residential exit instead. +on: + schedule: + # Once a day, off the hour, so it is not competing with everyone else's + # on-the-hour cron jobs for runner availability. + - cron: "23 6 * * *" + workflow_dispatch: + +jobs: + live-run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Playwright engine + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-playwright.txt + playwright install --with-deps chromium + + - name: Scrape a real, known-good search listing + id: run + # The same filtered search URL the README uses as its worked example, + # so a 0-result run means either the site changed or this address was + # refused — not "this URL never had products". Filtered (`&i=`) rather + # than a bare query, because a department-scoped search is the shape a + # real monitoring job uses. + # + # --pages 3, not 1: with one page pagination is never exercised at + # all, so a renamed next-page class (or the ?page= convention + # breaking) would stay invisible here — which is precisely the silent + # decay this workflow exists to catch. Amazon keeps serving a valid + # page for any page number, so the run's own "this page added nothing + # new" terminator is what has to be trusted, and three pages is enough + # to exercise it. + # + # continue-on-error: the exit code is read explicitly below so the + # interpretation (parser break vs. probable address block) can be + # printed before the job goes red. + # + # set +e: GitHub Actions runs `run:` steps as `bash -e`, so without + # this a non-zero scraper exit aborts the script on THAT line and the + # `echo "exit_code=$?"` below never runs. continue-on-error only stops + # the step from failing the job; it does not touch the shell's errexit. + continue-on-error: true + run: | + set +e + python3 playwright_scraper.py \ + --url "https://www.amazon.com/s?k=bluetooth+headphones&i=electronics" \ + --pages 3 --out canary_run --dump-html canary_run.html + # --proxy "${{ secrets.AMAZON_PROXY }}" # uncomment once the secret is set + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + + - name: Interpret the exit code + run: | + code="${{ steps.run.outputs.exit_code }}" + case "$code" in + 0) echo "OK — products parsed from a live run." ;; + 3) echo "::error::Blocked before parsing (exit 3): Amazon's image captcha, an unresolved 503 throttle, or a sign-in wall. Check canary_run.meta.json's stop_reason in the artifacts, and note this may be this runner's datacenter address rather than a site change — see AMAZON_PROXY in this workflow's header." ;; + 4) echo "::error::Ran fine but parsed 0 rows (exit 4) on a URL known to have plenty — this is the shape a real site-side break takes." ;; + 6) echo "::error::Partial run (exit 6) — some pages were gathered and then the run stopped early. canary_run.meta.json names which pages failed." ;; + 2) echo "::error::Bad usage (exit 2) — the workflow's own arguments are wrong, not the site." ;; + *) echo "::error::Unexpected exit code $code (see the step above for the traceback)." ;; + esac + test "$code" = "0" + + - name: Sanity-check the scraped data + # Exit 0 only means "some rows were written". It says nothing about + # whether the page actually rendered: a snapshot taken too early still + # writes rows, just with fields empty. Every floor below is set from a + # measured figure and deliberately well under it, so this fails on a + # collapse rather than on normal variation. + run: | + python3 - <<'EOF' + import json, sys + from pathlib import Path + + PAGES_REQUESTED = 3 + # Measured 2026-09-08 from a residential European exit: 3 pages of + # this URL gave 90 rows, 77 after duplicates were dropped, at 24-30 + # organic tiles per page. The floor is set at half of that. + MIN_ROWS = 40 + # Measured price coverage on this URL: 71-83%. Amazon genuinely + # withholds the price on roughly a quarter of tiles, so the floor is + # 50% and not the 90% a site that always prints a price would take. + MIN_PRICED_SHARE = 0.5 + + rows = json.loads(Path("canary_run.json").read_text()) + meta = json.loads(Path("canary_run.meta.json").read_text()) + fail = [] + + if len(rows) < MIN_ROWS: + fail.append(f"only {len(rows)} rows, expected at least {MIN_ROWS} " + f"across {PAGES_REQUESTED} pages") + + priced = sum(1 for r in rows if r.get("price") is not None) + if rows and priced / len(rows) < MIN_PRICED_SHARE: + fail.append(f"only {priced}/{len(rows)} rows have a price " + f"(floor {MIN_PRICED_SHARE:.0%}) — either Amazon is " + f"withholding far more than usual, or the price node " + f"moved") + + # Where the price came from. Every priced row on every measured run + # read `.a-price .a-offscreen`; a run that fell back to reassembling + # the visually split price for most rows means that node changed, and + # the numbers are worth re-checking even though they are not wrong. + if priced: + offscreen = sum(1 for r in rows if r.get("price_source") == "offscreen") + if offscreen / priced < 0.9: + fail.append(f"only {offscreen}/{priced} priced rows came from " + f"the .a-offscreen node — the price markup has " + f"probably changed") + + # Fields that were 100% populated on every page measured. Any of them + # collapsing means an anchor moved. + for field, floor in (("title", 0.95), ("rating", 0.8), + ("review_count", 0.8), ("image_url", 0.95), + ("sku", 1.0), ("category", 1.0)): + got = sum(1 for r in rows if r.get(field) not in (None, "")) + if rows and got / len(rows) < floor: + fail.append(f"{field} present on only {got}/{len(rows)} rows " + f"(floor {floor:.0%})") + + # A review_count is a count, not a rating with its digits glued on. + # This exact shape shipped once: "4.4 out of 5 stars, 279,961 + # ratings" read as 445279961. + absurd = [r["sku"] for r in rows + if isinstance(r.get("review_count"), int) + and r["review_count"] > 50_000_000] + if absurd: + fail.append(f"{len(absurd)} row(s) have an implausible " + f"review_count (>50M): {absurd[:3]} — the count is " + f"being read from a node that also holds the rating") + + # The point of running more than one page. + if meta.get("pages_completed", 0) < PAGES_REQUESTED: + fail.append(f"pagination stopped after {meta.get('pages_completed')} " + f"of {PAGES_REQUESTED} pages (stop_reason=" + f"{meta.get('stop_reason')!r}) — the next-page class or " + f"the ?page= convention has probably broken") + if meta.get("status") != "complete": + fail.append(f"run status is {meta.get('status')!r}, not 'complete'") + if meta.get("mode") != "listing": + fail.append(f"run mode is {meta.get('mode')!r}, not 'listing'") + if meta.get("source") != "amazon.com": + fail.append(f"run source is {meta.get('source')!r} — the URL was " + f"redirected to another marketplace") + + for line in fail: + print(f"::error::{line}") + print(f"[canary] {len(rows)} rows, {priced} priced, " + f"status={meta.get('status')}, " + f"pages={meta.get('pages_completed')}/{PAGES_REQUESTED}") + sys.exit(1 if fail else 0) + EOF + + - name: Upload run artifacts (output, HTML snapshot, debug dumps) + # Always, not just on failure: a run that returns the WRONG numbers + # with no error is exactly the regression that needs the actual bytes + # the parser saw, not a pass/fail badge. + if: always() + uses: actions/upload-artifact@v4 + with: + name: canary-run-${{ github.run_id }} + path: | + canary_run.json + canary_run.csv + canary_run.meta.json + canary_run.html* + canary_run_page*_debug.* + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 0000000..37e66f3 --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,45 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, ready_for_review, reopened] + # Optional: Only run on specific file changes + # paths: + # - "src/**/*.ts" + # - "src/**/*.tsx" + # - "src/**/*.js" + # - "src/**/*.jsx" + +jobs: + claude-review: + # Optional: Filter by PR author + # if: | + # github.event.pull_request.user.login == 'external-contributor' || + # github.event.pull_request.user.login == 'new-developer' || + # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code Review + id: claude-review + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' + plugins: 'code-review@claude-code-plugins' + prompt: '/code-review:code-review --comment ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' + claude_args: '--allowedTools "mcp__github_inline_comment__create_inline_comment"' + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000..6b15fac --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,50 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + # claude_args: '--allowed-tools Bash(gh pr *)' + diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..da22063 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,214 @@ +name: tests + +# Runs the offline suite only. Nothing here touches amazon.com, 2captcha.com +# or any credential — GitHub Actions has no key, and a CI job that hits a live +# site fails for reasons that have nothing to do with the commit. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + offline: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # 3.9 is the floor the README claims. Claiming it without testing it is + # how a walrus operator or a `X | None` annotation ships and breaks it. + python-version: ["3.9", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install core dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Byte-compile everything + # Catches a syntax error on the oldest supported Python before the + # suite gets a chance to skip the file. + run: python -m compileall -q . + + - name: Offline test suite + # Prints its own count. No network, no key, ~1s. + run: python smoke_test.py + + - name: Offline test suite, via pytest + # tests/test_smoke.py wraps the same run above for the pytest entry + # point pyproject.toml declares — this step exists so THAT wrapper + # can't silently rot even though the line above already covers the + # actual checks. + run: | + pip install pytest + pytest -q + + - name: Every CLI answers --help + # A crash in argparse setup or a bad local import is invisible to the + # test suite but breaks the first thing any user types. The engine + # libraries are deliberately NOT installed here, so an ImportError + # naming one of them is expected; anything else is a failure. + run: | + python - <<'EOF' + import subprocess, sys + + ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager") + failed = [] + + for script in ["playwright_scraper.py", "puppeteer_scraper.py", + "selenium_scraper.py", "scraper_api_client.py", + "fingerprint_client.py", "env_config.py", + "diff_runs.py"]: + r = subprocess.run([sys.executable, script, "--help"], + capture_output=True, text=True) + if r.returncode == 0: + print(f"ok {script}") + continue + blob = r.stdout + r.stderr + if "ModuleNotFoundError" in blob and any(l in blob for l in ENGINE_LIBS): + print(f"skipped {script} (engine library not installed here)") + continue + print(f"FAILED {script}\n{blob}") + failed.append(script) + + sys.exit(1 if failed else 0) + EOF + + - name: Committed sample output is real + # The README links to these two files, so a push without them ships + # broken links. And a sample invented by hand drifts from the schema and + # teaches readers the wrong field names, so these are cut from a real + # run rather than written by hand. + run: | + python - <<'EOF' + import csv, json, sys + from pathlib import Path + + MARKERS = ("sample-product-", "example brand", "sample product", + "product description text", "lorem ipsum") + + fail = [] + for name in ("sample_output.json", "sample_output.csv"): + if not Path(name).is_file(): + fail.append(f"{name} is missing — regenerate it from a real run") + + if not fail: + rows = json.loads(Path("sample_output.json").read_text()) + if not rows: + fail.append("sample_output.json is empty") + blob = json.dumps(rows).lower() + hit = [m for m in MARKERS if m in blob] + if hit: + fail.append(f"sample_output.json looks fabricated: {hit}") + + # Column set must match what the writers actually produce. + from output_writer import Product + from dataclasses import asdict + expected = list(asdict(Product()).keys()) + for i, row in enumerate(rows): + if list(row.keys()) != expected: + fail.append(f"row {i} columns drifted from output_writer.Product") + break + + with open("sample_output.csv", newline="", encoding="utf-8") as f: + header = next(csv.reader(f)) + if header != expected: + fail.append("sample_output.csv header drifted from output_writer.Product") + + for line in fail: + print("FAILED:", line) + sys.exit(1 if fail else 0) + EOF + + - name: Config reporter runs with no .env present + run: python env_config.py + + - name: No credentials committed + # A key pasted into a file during debugging is the one mistake in this + # repo that cannot be undone by a later commit. + run: | + set -e + if grep -rInE '(ws|wss)://[^ "'"'"']+:[^ "'"'"']+@' \ + --include='*.py' --include='*.md' --include='*.txt' --include='*.yml' . \ + | grep -vE 'USER:PASS|user:pass|ACCOUNT:PASSWORD|\{login\}|\*\*\*|password\}@'; then + echo "FAILED: what looks like a real credentialled URL is committed" + exit 1 + fi + if grep -rIn --include='*.py' --include='*.md' -E '\b[0-9a-f]{32}\b' . \ + | grep -viE 'sha|hash|nonce|example|md5'; then + echo "FAILED: what looks like a 2captcha API key is committed" + exit 1 + fi + echo "clean" + + engine-smoke: + # The offline job above installs NO engine library on purpose (that is what + # makes it offline), so every engine-guarded check in smoke_test.py skips + # in it — and "skipped, engine absent" reads exactly like a passing run. + # This job makes those checks actually execute. + # + # ONE VENV PER ENGINE, and not `pip install playwright pyppeteer selenium` + # into one environment, which is what this started as. Two reasons, and the + # second is why it matters: + # + # * The three engines' pins are mutually unsatisfiable (playwright wants + # pyee>=13, pyppeteer wants pyee<12; pyppeteer wants urllib3<2, + # selenium wants >=2.6), which is exactly what the README tells users, + # so a venv each is also what a user actually does. + # * Installed together and UNPINNED, pip resolved the conflict by + # reaching for pyppeteer 0.0.25 — a stub release nobody uses — instead + # of the >=1.0.2 requirements-puppeteer.txt asks for. CI was green + # against a version the repo does not support. Installing from the + # requirements files, separately, tests the versions users get. + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + engine: [playwright, puppeteer, selenium] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install core + the ${{ matrix.engine }} engine, in its own venv + # No browser binaries: the offline suite imports these libraries and + # never launches one. + run: | + python -m venv .venv + .venv/bin/python -m pip install --upgrade pip + .venv/bin/pip install -r requirements.txt \ + -r requirements-${{ matrix.engine }}.txt + echo "--- what actually got installed ---" + .venv/bin/pip list --format=freeze | grep -iE '^(playwright|pyppeteer|selenium|pyee|urllib3|beautifulsoup4)=' + + - name: The requirements file is self-consistent + # Passes here precisely BECAUSE each engine has its own venv. Run in a + # shared one it would report the three-way conflict above, which is the + # thing this layout exists to avoid. + run: .venv/bin/pip check + + - name: Offline suite, with ${{ matrix.engine }} actually importable + # The suite names each engine it could not import. In this job that + # engine IS installed, so naming it means a real import error rather + # than an expected absence — the other two engines are absent here by + # design and their skips are fine. + run: | + set -o pipefail + .venv/bin/python smoke_test.py | tee smoke.log + if grep -q "${{ matrix.engine }}_scraper could not be imported" smoke.log; then + echo "FAILED: ${{ matrix.engine }}_scraper failed to import even though" + echo " ${{ matrix.engine }} is installed in this job." + exit 1 + fi + + - name: The ${{ matrix.engine }} CLI answers --help + run: .venv/bin/python ${{ matrix.engine }}_scraper.py --help >/dev/null diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2363300 --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +# Everything below is GENERATED by running this project. None of it belongs in +# git — it is either large, machine-specific, or a snapshot of a site that will +# have changed by the time anyone reads it. A single 3-page Amazon run drops +# ~5 MB of HTML snapshots alone. + +# --- scrape output ------------------------------------------------------- +*.json +*.csv +!package.json +# Committed on purpose: a real 5-row excerpt of a real run, so a reader can see +# the exact output shape without installing anything. +!sample_output.json +!sample_output.csv +amazon_products.* +amazon_products_*.* + +# --- the live-test harness's working directory --------------------------- +live_results/ +live_results.zip + +# --- debug artifacts the engines drop next to themselves ----------------- +# Written on purpose when a page yields nothing or comes back blocked, and by +# --dump-html on success too. Large, and stale within hours. +*_debug.html +*_debug.png +*.html.page* + +# --- captures kept for parser work --------------------------------------- +# The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); +# the multi-megabyte originals they were cut from are not. +captures/ + +# --- the landing page and the proxy list --------------------------------- +# Both live outside the repo on purpose: the landing page is published +# elsewhere, and a downloaded 2captcha proxy list is a credential file. +index.html +proxylist.txt + +# --- caches -------------------------------------------------------------- +__pycache__/ +*.py[cod] +.pytest_cache/ +.fingerprint_cache/ + +# --- packaging build artifacts -------------------------------------------- +*.egg-info/ +dist/ +build/ + +# --- environments -------------------------------------------------------- +.venv/ +venv/ +env/ + +# --- editors / OS -------------------------------------------------------- +.DS_Store +*.swp +.idea/ +.vscode/ + +# --- never, under any circumstances -------------------------------------- +# The scrapers mask user:pass in their own logs, but raw HTML dumps and the +# Scraper API's x-debug header are NOT masked, and a key pasted into a shell +# history file is not masked either. +*.log +.env +*.key +secrets* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4a59d06 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,94 @@ +# Changelog + +All notable changes to this project are documented here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and versions follow [Semantic Versioning](https://semver.org/) as closely as a +CLI toolkit can. Read a **patch** as "fixes" rather than as a promise that +every flag and default is frozen: a default that changes behaviour can ship in +one, and when it does the release notes say so first. + +## [0.1.0] — 2026-09-08 + +First release. The repository previously held a different, unrelated Amazon +scraper (a `requests` + `lxml` client with its own output format); this +version replaces it entirely with the architecture used across this family of +scrapers. **Nothing from the previous code is source- or output-compatible.** + +### Added + +- **Three modes.** `--mode listing` (default) reads paginated search results + and best-seller grids; `--mode product` reads one `/dp/{ASIN}` page with + brand, seller, bullets, images and availability; `--mode reviews` reads the + reviews Amazon renders to a visitor with no account. +- **All 21 marketplaces**, selected by the hostname of `--url`. There is no + `--marketplace` flag, so a flag and a URL cannot disagree. +- **Four engines**: `playwright_scraper.py` (primary), `selenium_scraper.py`, + `puppeteer_scraper.py`, and `scraper_api_client.py` (browserless, via the + 2Captcha Scraper API). All of them agree on exit codes and run status. +- **The family's output contract**: the `Product` row schema with the shared + sixteen columns first, JSON and CSV in the same field order, a + `.meta.json` run sidecar, and exit codes 0/1/2/3/4/5/6. +- **`page_flow.py`**, a shared page-state policy, so the three engines cannot + drift on what a page means. Amazon answers a request in five ways and four + of them want different responses: + - the **AWS WAF JavaScript challenge** (HTTP 202) is waited out, not + reported and not solved — a browser clears it in about four seconds, and + handing it to a captcha solver would bill for nothing; + - the **503 throttle** is retried, then answered with a different exit; + - Amazon's **image captcha** is the only path a 2Captcha key is for; + - a **sign-in redirect** stops the run, because no proxy or solve changes + it. +- **`--concurrency N`** in the Playwright engine, one browser and one exit per + worker, with page 1 always fetched alone. +- **`--price-tolerance-pct`** in `diff_runs.py`, for runs whose exit country + differs from the marketplace: Amazon converts the price and the rate drifts, + measured at 0.04% over ten minutes on 10 of 43 ASINs. Defaults to 0, so + every cent is reported unless you ask otherwise. +- **`--chromium-path`** in the pyppeteer engine, for platforms where its own + bundled Chromium will not start (an x86_64 build under Rosetta on Apple + Silicon). +- **A `page` column** beside `position`, because Amazon's `data-index` + restarts on every page and position alone is ambiguous across a paginated + run. +- 239 offline checks in `smoke_test.py`, with fixtures cut from real captures; + a daily canary against a real 3-page listing; `tests.yml` on Python 3.9 and + 3.12 with an `engine-smoke` job that fails on any unexpected skip. + +### Fixed before release + +- **`engine-smoke` was green against a stub.** The job installed the three + engines unpinned into one environment, and pip resolved their mutually + unsatisfiable pins by reaching for **pyppeteer 0.0.25** — not the `>=1.0.2` + `requirements-puppeteer.txt` asks for. It now installs each engine from its + own requirements file into its own venv, which is both what the README tells + users to do and the only way to get the versions they get; `pip check` runs + per venv as well. +- **The pyppeteer import was inside the launch path**, so `puppeteer_scraper` + imported cleanly with no pyppeteer installed at all. That made the offline + suite's skip reporting — and therefore CI's "fail on any skip" guard — + vacuous for that engine, and is why the stub above went unnoticed. The + import is now at module level like its two siblings, and a new check asserts + all three stay that way. + +### Notes on what this does NOT do + +- **No JSON-LD parsing**, because Amazon publishes none — zero + `application/ld+json` blocks on any page kind measured. Rows come from the + site's own `data-asin` attributes, with a `/dp/{ASIN}` URL-pattern fallback. +- **No `brand` on listing rows.** Amazon's search markup has no brand field; + guessing from the title would be wrong often enough to be worse than a null. + `--mode product` does populate it. +- **No `prime` column.** No Prime marker appeared on any tile captured from a + cross-border exit, so the column would have been null on every row. +- **No `--pages` for `--mode reviews`.** `/product-reviews/{ASIN}` redirects + to `/ap/signin` on page 1, so the paginated history is not available to an + anonymous visitor at all. What this mode returns is the ~13 reviews the + product page itself renders. +- **No logged-in scraping.** Reviews beyond those, and order or cart data, are + out of scope. +- **The image-captcha solve path is unproven against the live site.** No run + during development was served one; it is exercised only by an offline + fixture written from Amazon's documented markup. + +[0.1.0]: https://github.com/2scraper/amazon-scraper/releases/tag/v0.1.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..08cfb8b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +support@2captcha.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..99480b9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,115 @@ +# Contributing + +Bug reports, site-change reports and pull requests are all welcome. This file +covers the few things specific to a scraper, which are not the usual ones. + +## Before you open anything + +Run the offline suite. It needs no network, no browser and no API key, and takes +about a second: + +```bash +pip install -r requirements.txt +python3 smoke_test.py +``` + +It prints its own check count, and lists any group it had to skip because an +engine library is absent. + +**The suite must pass with no engine installed at all.** CI installs only +`beautifulsoup4` and `requests`, so any import of `playwright_scraper`, +`puppeteer_scraper` or `selenium_scraper` in a test has to sit inside +`try/except ImportError` with the skip recorded. This is easy to get wrong +locally, where you almost certainly have an engine installed and an unguarded +import passes. + +If the suite fails on a clean clone, that is itself the bug — say so. + +## Never commit a credential + +`.env` is in `.gitignore`. Keep it there. + +The scrapers mask `user:pass@` in their own log lines, but three things are **not** +masked: raw HTML dumps, the Scraper API's `x-debug` response header, and your +shell history. Before pasting any output into an issue or a PR, replace keys, +proxy passwords and full `ws://user:pass@host:9222` endpoints with `***`. + +CI fails the build if something that looks like a credential is committed. That +check is a backstop, not a review — a leaked key has to be rotated whether or +not the check caught it. + +## Reporting a site change + +Amazon changing its markup is the normal way this stops working, and it has its +own issue template. The one detail that saves the most time: the parser tries +**JSON-LD first**, then a CSS + URL-pattern fallback. Knowing which of the two +broke narrows the fix immediately. `--out dump` writes the page next to the +output when a run finds nothing. + +## Pull requests + +**Add a test for the behaviour you are changing.** `smoke_test.py` is a single +file of plain functions with inline HTML/JSON fixtures — no pytest, no +conftest, no fixtures directory. Copy the nearest existing check and edit it. + +Four properties in this repo exist because they were once absent and cost real +time. Tests pin all four, so a PR that breaks one will fail rather than +silently regress: + +- **The product URL comes from `offers.url`, not `node.url`.** No product on + this site carries `node.url`. Read the wrong field and every row points at the + category page while title, brand and price all look correct. +- **A run that finds nothing writes nothing.** It must not replace a good output + file with `[]`. `--allow-empty` is the opt-out. +- **Exit codes are a contract**, not decoration: `0` ok, `1` crash, `2` bad + usage, `3` blocked by a challenge, `4` zero products, `5` remote API error, + `124` self-imposed timeout. A pipeline branches on these. +- **A sku already written by an earlier page of the same run is dropped, not + duplicated.** All three browser engines paginate by following + `NEXT_PAGE_SELECTOR`; a stale or repeating link must not double a row in the + output. See `dedupe_by_sku` in `output_writer.py` and `diff_runs.py`, which + diffs two runs by the same key. + +There is also a naming check: certain phrases are banned repo-wide and the suite +fails naming them. If it trips, read the message — the phrase is wrong for a +reason, not merely unfashionable. + +### Style + +- **Match the file you are editing.** No formatter is enforced. +- **Comments explain *why*.** What the code does is visible; why it does it that + way, especially where the obvious version is wrong, is not. +- **A timeout on every remote call.** Every browser library used here has needed + an explicit timeout its own API does not provide, and each has needed its own + route out of the runtime — reporting a timeout is not the same as exiting on + one. If you add a call to a remote browser or API, bound it. +- **Fail loudly.** A function that returns an empty list on error, or logs + success without checking that the thing it wanted actually happened, is the + single most common bug class in this codebase's history. A selector that + matches the *wrong* element is worse than one that matches nothing, because + the second one tells you. + +### If your change needs a live run + +Most do not — the suite covers the parser, the writers, the captcha classifier +and the CLI contract against inline fixtures. If yours genuinely needs +amazon.com, say in the PR what you ran, on which marketplace, from which +exit country, and what you +got. Product counts differ by country and by URL, so a bare "worked for me" is +not reproducible. + +Do not add anything that submits the registration form. This project +deliberately never does, and a captcha token proved valid by creating a real +account is not a result worth having. + +## Scope + +This repo scrapes **public pages** on Amazon: search results, best-seller +grids, product pages, and the reviews shown to a visitor with no account. +Out of scope: +anything behind a login, anything that submits a form, and anything that +defeats a protection rather than passing it the way an ordinary browser does. + +## Licence + +MIT. By opening a pull request you agree your contribution ships under it. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..922b993 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# Builds the Playwright engine (the one the README recommends) into a +# container with its own Chromium — for a CI canary run or a scheduled job, +# not required for local development (`pip install` directly is simpler there). +# +# docker build -t amazon-scraper . +# docker run --rm -v "$PWD/out:/out" amazon-scraper \ +# --url "https://www.amazon.com/s?k=wireless+headphones" \ +# --pages 3 --out /out/headphones +# +# Pass --proxy/--twocaptcha-key the same way as running locally, or mount a +# .env at /app/.env — nothing here bakes in a credential. +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt requirements-playwright.txt ./ +RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.txt \ + # Playwright's own apt-get for Chromium's shared-library dependencies — + # not pip packages, so this has to run as a separate, explicit step. + && playwright install --with-deps chromium + +COPY captcha_solver.py env_config.py fingerprint_client.py output_writer.py \ + page_flow.py playwright_scraper.py product_parser.py diff_runs.py ./ + +ENTRYPOINT ["python3", "playwright_scraper.py"] +CMD ["--help"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..18f8a81 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 2Captcha + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index a6d4824..97b4ae5 100644 --- a/README.md +++ b/README.md @@ -1,301 +1,474 @@ -# Amazon Scraper — 2parser +# amazon-scraper -![Python](https://img.shields.io/badge/Python-3.9%2B-blue?logo=python&logoColor=white) -![Marketplaces](https://img.shields.io/badge/Marketplaces-21-orange) -![Proxies](https://img.shields.io/badge/Proxies-2captcha.com-green) -![License](https://img.shields.io/badge/License-MIT-lightgrey) +[![release](https://img.shields.io/github/v/release/2scraper/amazon-scraper?sort=semver)](https://github.com/2scraper/amazon-scraper/releases) +[![tests](https://github.com/2scraper/amazon-scraper/actions/workflows/tests.yml/badge.svg)](https://github.com/2scraper/amazon-scraper/actions/workflows/tests.yml) +[![canary](https://github.com/2scraper/amazon-scraper/actions/workflows/canary.yml/badge.svg)](https://github.com/2scraper/amazon-scraper/actions/workflows/canary.yml) +[![Python](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/) +[![licence](https://img.shields.io/badge/licence-MIT-lightgrey)](LICENSE) +[![engines](https://img.shields.io/badge/engines-Playwright%20%7C%20Selenium%20%7C%20pyppeteer%20%7C%20CDP-informational)](#engines) +[![runs without an account](https://img.shields.io/badge/runs%20without-an%20account-brightgreen)](#what-works-with-nothing-at-all) -Production-grade Amazon scraper with support for all 21 marketplaces and [2captcha.com](https://2captcha.com/proxy) proxy rotation. +Scrapes Amazon search results, best-seller grids, product pages and the +reviews Amazon shows to visitors with no account — on any of its 21 +marketplaces. JSON or CSV, one row schema per mode, and a run-metadata +sidecar that says whether the result is complete. + +Four engines: **Playwright** (primary), **Selenium**, **pyppeteer**, or a +remote browser over CDP such as the **Scraping Browser API**. --- -## Features +## What works with nothing at all + +The most useful thing this README can tell you is where the paid path is +unnecessary. Measured on **2026-09-08**, from a residential exit IP in +Europe, with a plain local Chromium — **no API key, no proxy, no account**: + +``` +$ python3 playwright_scraper.py \ + --url "https://www.amazon.com/s?k=bluetooth+headphones&i=electronics" \ + --pages 3 + +Parsed 30 rows from page 1. Price coverage on page 1: 24/30 (80%). +Parsed 30 rows from page 2. Price coverage on page 2: 22/30 (73%). +Parsed 30 rows from page 3. Price coverage on page 3: 25/30 (83%). +Saved 77 products -> amazon_products.json # 90 rows, 13 duplicates dropped +Wrote run metadata -> amazon_products.meta.json (status=complete) +$ echo $? +0 +``` + +Amazon's front door met that run twice and neither time cost anything: an +**AWS WAF JavaScript challenge**, which a real browser solves by itself +(measured: 3.9 s on amazon.co.uk), and a **503 throttle** on amazon.de, which +a retry cleared. + +So what do the 2Captcha products actually buy here? -- **21 marketplaces** — .com, .de, .co.uk, .fr, .co.jp, .in, .com.br and more -- **2captcha proxies** — native format, round-robin rotation, health tracking, ban cooldown -- **Search** — pagination, price/sort filters, automatic missing price fill -- **Products** — price, rating, images, bullet points, variations, seller info -- **Best Sellers** — 4 DOM strategies including current 2024–2025 layout -- **Reviews** — author, rating, body, verified purchase, helpful votes -- **Anti-bot** — 4 browser fingerprint profiles, per-market Accept-Language, 503/429 backoff -- **Export** — JSON, JSON Lines, CSV (UTF-8-BOM, opens in Excel without conversion) +| Product | What it is for on Amazon | +|---|---| +| **Proxies** (`--proxy-file`) | Volume. One address doing 50 pages an hour gets scored; twenty doing five each do not. Also the only way to choose an exit country. | +| **Scraping Browser API** (`--cdp-endpoint`) | A browser you do not run or patch, with a persistent profile and a chosen exit country. Also `Captcha.setAutoSolve` inside the browser. | +| **Captcha solving** (`--twocaptcha-key`) | One page: Amazon's own image captcha, "Enter the characters you see below". Not the WAF challenge, not a 503, not a sign-in wall. | +| **Fingerprints** (`--fingerprint`) | A consistent device identity across runs, matched to the exit country. | --- -## Installation +## Install ```bash -git clone https://github.com/2parser/amazon-scraper +git clone https://github.com/2scraper/amazon-scraper cd amazon-scraper -pip install -r requirements.txt +pip install -r requirements.txt # core: beautifulsoup4, requests + +# then ONE engine +pip install -r requirements-playwright.txt && playwright install chromium +# or pip install -r requirements-selenium.txt (needs a local Chrome) +# or pip install -r requirements-puppeteer.txt (downloads its own Chromium) ``` +**Install exactly one engine.** playwright and pyppeteer declare +mutually unsatisfiable pins (`pyee` <12 vs ≥13), and pyppeteer and selenium +collide on `urllib3` (<2.0 vs ≥2.6). All three do run side by side in +practice, because neither library touches the incompatible part — but +`pip check` reports the conflict and pip may resolve it by downgrading +something you wanted. Use a virtualenv per engine if you need more than one. + --- -## Proxies +## Three modes + +```bash +# 1. LISTING (default) — search results or a best-seller grid, paginated +python3 playwright_scraper.py --url "https://www.amazon.com/s?k=wireless+headphones" --pages 3 +python3 playwright_scraper.py --url "https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics/" --pages 2 -Proxies are taken from your **2captcha.com personal account → Proxy** section. -Download the list and pass it via `--proxies`: +# 2. PRODUCT — one /dp/{ASIN} page, with brand, seller, bullets and images +python3 playwright_scraper.py --mode product --url "https://www.amazon.com/dp/B07K5214NZ" +# 3. REVIEWS — the reviews Amazon renders to a visitor with no account +python3 playwright_scraper.py --mode reviews --url "https://www.amazon.com/dp/B07K5214NZ" ``` -# proxies.txt — one proxy per line (2captcha account format) -http://host:port:login:password -http://host:port:login:password -``` -> No API key required for proxies — they come with login/password credentials. -> The API key (`--2captcha-key`) is only needed for CAPTCHA solving (optional). +The marketplace comes from the URL's hostname. There is no `--marketplace` +flag, deliberately: one fewer way for a flag and a URL to disagree about +which site a run is reading. + +### Why `--mode reviews` has no `--pages` + +`/product-reviews/{ASIN}` — the paginated review history — **redirects to +`/ap/signin` on page 1**, not on page 2. An anonymous visitor cannot read it +at all. What this mode returns is the reviews rendered on the product page +itself: **13 per product** in every measurement here, complete with author, +rating, date, verified-purchase flag, helpful votes and body text. There is +no second page to fetch, so there is no flag to ask for one. + +If you need the full history, you need an account, and this repo does not go +there: scraping from a logged-in session risks the account, and a canary +could not cover the path. --- -## Quick Start +## Output + +One row per product, same field order in JSON and CSV. The **first sixteen +columns are shared with every other scraper in this family**, so a consumer +written against one of them reads these unchanged; Amazon's own columns come +after. + +| Column | Notes | +|---|---| +| `source` | The marketplace host the row came from: `amazon.com`, `amazon.co.jp`, … | +| `scraped_at` | UTC, ISO 8601 | +| `url` | Rebuilt as `https://www.{host}/dp/{ASIN}` — see [below](#the-url-column-is-rebuilt-not-read) | +| `sku` | The ASIN | +| `title`, `brand`, `price`, `currency`, `original_price`, `discount_pct` | `brand` is null in listing mode — see [the traps](#traps-that-look-like-bugs) | +| `rating`, `review_count`, `in_stock`, `image_url`, `category` | | +| `price_source` | `offscreen` / `split` / `detail` — which node the price was read from | +| `page`, `position` | Which listing page, and the position within it. On a best-seller grid `position` is Amazon's published rank (#1–#50) | +| `sponsored`, `badge`, `coupon` | Sponsored placements are flagged, not dropped | +| `seller`, `availability`, `bullets`, `images`, `variations` | `--mode product` only; null on a listing run | + +`--mode reviews` writes a different schema: `sku`, `review_id`, `title`, +`rating`, `author`, `review_date`, `verified_purchase`, `helpful_votes`, +`variant`, `variant_asin`, `body`. + +See [`sample_output.json`](sample_output.json) and +[`sample_output.csv`](sample_output.csv) — both cut from a real run, not +written by hand. CI checks their columns against the schema. + +### The run-metadata sidecar + +Every successful run writes `.meta.json` beside its output: + +```json +{ + "source": "amazon.com", + "mode": "listing", + "status": "complete", + "stop_reason": "completed", + "pages_requested": 3, + "pages_completed": 3, + "pages_failed": [], + "products": 77, + "start_url": "https://www.amazon.com/s?k=bluetooth+headphones&i=electronics", + "final_url": "https://www.amazon.com/s?k=bluetooth+headphones&i=electronics&page=3", + "finished_at": "2026-09-08T13:23:23.751597+00:00" +} +``` + +`status` is the field to branch on: `complete`, `partial` or `failed`. +`pages_failed` names WHICH pages produced nothing, by number — a count stops +being a description once page 3 can fail while 4 and 5 succeed. -```bash -# Search — 10 pages on amazon.de, fill missing prices -python main.py search "iphone" -m de \ - --proxies proxies.txt \ - --pages 10 \ - --fill-prices \ - --output results.json +A **failed** run writes no sidecar at all, because it also does not overwrite +the previous run's output, and a `"failed"` sidecar sitting beside good data +would contradict it. -# Single product by ASIN -python main.py product B08N5WRWNW -m de --proxies proxies.txt +### Exit codes -# Best Sellers -python main.py best-sellers "Electronics" -m de --proxies proxies.txt +| Code | Meaning | +|---|---| +| 0 | Rows written | +| 1 | Crash | +| 2 | Bad usage (including a credentialed `--cdp-endpoint` given to Selenium) | +| 3 | Blocked before parsing: Amazon's image captcha, an unresolved throttle, or a sign-in wall | +| 4 | Ran fine, found nothing | +| 5 | Remote API error | +| 6 | Partial: some pages gathered, then the run stopped early | -# Reviews -python main.py reviews B08N5WRWNW -m de --proxies proxies.txt --pages 5 +Exit 3 covers the sign-in wall as well as a captcha, because the family's +codes have one slot for "something stood between the run and the content". +The sidecar's `stop_reason` (`needs_signin` vs `blocked_amazon-captcha`) +keeps them apart for anyone who needs to know which it was. -# Bulk scraping from ASIN list file (one ASIN per line) -python main.py bulk asins.txt -m de --proxies proxies.txt --workers 4 +### A run that finds nothing writes nothing -# List all supported marketplaces -python main.py --list-marketplaces -``` +By default, zero rows means **no file is written** and the exit code is 4. +That is deliberate: a page-load failure that writes `[]` over last night's +good output destroys the last known good data, and a consumer cannot tell an +empty result from a failed run. Pass `--allow-empty` when an empty result is +genuinely the answer you want recorded. --- ## Marketplaces -`-m` / `--marketplace` accepts any format: +All 21, selected by the URL you pass: -```bash --m de # TLD --m uk # alias → co.uk --m co.jp # full TLD --m amazon.fr # domain --m https://www.amazon.it/dp/... # URL — marketplace auto-detected -``` +`amazon.com` · `co.uk` · `de` · `fr` · `it` · `es` · `nl` · `com.be` · `pl` · +`se` · `co.jp` · `ca` · `com.au` · `in` · `com.br` · `com.mx` · `sg` · `ae` · +`sa` · `com.tr` · `eg` -| TLD | Country | Currency | -|-----|---------|----------| -| com | United States | USD | -| co.uk | United Kingdom | GBP | -| de | Germany | EUR | -| fr | France | EUR | -| it | Italy | EUR | -| es | Spain | EUR | -| nl | Netherlands | EUR | -| pl | Poland | PLN | -| se | Sweden | SEK | -| co.jp | Japan | JPY | -| ca | Canada | CAD | -| com.au | Australia | AUD | -| in | India | INR | -| com.br | Brazil | BRL | -| com.mx | Mexico | MXN | -| sg | Singapore | SGD | -| ae | UAE | AED | -| sa | Saudi Arabia | SAR | -| com.tr | Turkey | TRY | -| com.be | Belgium | EUR | -| eg | Egypt | EGP | - -`Accept-Language` and proxy geo-targeting are automatically configured per marketplace. +A hostname not in that table still parses; it only loses the currency hint +used to disambiguate a bare local symbol. --- -## Python API - -```python -from scraper import AmazonScraper -from proxy_manager import ProxyManager -from output import save_json, save_csv - -# Initialize -scraper = AmazonScraper( - proxy="http://host:port:login:password", # single proxy - # proxy_list=["http://...", "http://..."], # or a list - marketplace="de", -) - -# Load proxies from file -scraper.proxy_manager = ProxyManager.from_file("proxies.txt") -scraper.client.proxy_manager = scraper.proxy_manager - -# Search -results = scraper.search( - "iphone", - pages=10, - sort_by="review_rank", # featured|price_asc|price_desc|review_rank|date_rank - fill_missing_prices=True, # extra /dp/ request for variative products -) -save_json([r.to_dict() for r in results], "results.json") -save_csv([r.to_dict() for r in results], "results.csv") - -# Single product -product = scraper.get_product("B08N5WRWNW") -print(product.title, product.price, product.currency) - -# Best Sellers -items = scraper.get_best_sellers("Electronics") - -# Reviews -reviews = scraper.get_reviews("B08N5WRWNW", pages=5) - -# Bulk -products = scraper.get_products_bulk(["B08N5WRWNW", "B07VGRJDFY"], workers=3) -``` +## Measured results + +Everything below is a number from a real run on **2026-09-08**, from a +European residential exit IP. Nothing here is an aspiration. + +| What | Measured | +|---|---| +| Search page, amazon.com | 16–30 organic tiles per page (it varies by query) | +| Search page, amazon.de | 16 tiles | +| Search page, amazon.co.jp | 60 tiles | +| Best-seller grid | 50 cards per page, ranks 1–100 across two pages, contiguous | +| Reviews on a product page | 13 | +| Price present, amazon.com search | 71–83% of rows (16/22, 24/30, 22/30, 25/30) | +| Price present, best-seller grid | 100% (50/50, 49/50) | +| `rating`, `review_count`, `image_url`, `title` | 100% of rows on every page measured | +| `brand`, listing mode | 0% — Amazon renders no brand field on a search tile | +| AWS WAF challenge clearing itself | 3.9 s | +| Pages of a 3-page run completed | 3 of 3, `status: complete`, exit 0 | --- -## Proxy formats +## Traps that look like bugs -All common formats are supported: +Every one of these was hit during development. A reader who meets one +unwarned concludes the tool is broken. -| Format | Example | -|--------|---------| -| 2captcha account (main) | `http://host:port:login:password` | -| HTTPS | `https://host:port:login:password` | -| SOCKS5 | `socks5://host:port:login:password` | -| No scheme | `host:port:login:password` | -| URL format | `login:password@host:port` | -| No auth | `host:port` | +### `brand` is null on every listing row ---- +Amazon's search markup has no brand field. The brand is inside the title text +and nowhere else. Splitting the title on its first word would be wrong often +enough to be worse than a null, so the column stays empty in listing mode. It +**is** populated by `--mode product`, from the byline (`Visit the ZIHNIC +Store` → `ZIHNIC`). -## Fill missing prices (`--fill-prices`) +### About a quarter of search rows have no price -Variative products (iPhones in different colors, laptops with different specs) -don't show a single price in search results — Amazon displays "More Options" instead. +On one measured page, 16 of 22 tiles carried a price and six did not — real +products (Sony WH-CH520, Bose QuietComfort) for which Amazon simply renders +no price to that visitor. `price`, `currency` and `price_source` are all null +on such a row, and the row is still kept. Each run logs its price coverage; +below 50% it warns, because that is low enough to suspect the snapshot rather +than the catalogue. -`--fill-prices` automatically makes an extra `/dp/ASIN` request for each such product: +### The currency follows the exit IP, not the domain -``` -INFO fill_missing_prices: 3 items without price → fetching product pages -INFO ✓ B0CHX96JDY → 788.00 EUR -INFO ✓ B09G995PVT → 303.00 EUR -INFO ✓ B0CHWWM3JH → 774.00 EUR -INFO fill_missing_prices: filled 3/3 items -``` +Served to a visitor whose delivery country differs from the marketplace, +Amazon **converts** the price. From a European exit, amazon.co.uk and +amazon.co.jp both quote `EUR`. So: ---- +* never derive the currency from the hostname — this scraper does not, and + `currency` is null rather than guessed when the page does not say; +* **the converted price drifts with the exchange rate.** Two runs of the same + command ten minutes apart disagreed on 10 of 43 shared ASINs, every one by + the same relative amount (0.039–0.058%: `25.80 → 25.81`, `51.61 → 51.63`, + `1117.53 → 1118.00`). Nothing about the offers changed; the rate ticked. + `diff_runs.py --price-tolerance-pct 0.1` classifies those as FX noise + instead of price changes. It defaults to 0 — report every cent — because a + run against a marketplace in its own currency has no conversion in it. + +### `--mode reviews` sometimes needs several tries + +Amazon serves two variants of the same `/dp/` URL: one with the reviews in +the markup, one where the widget is present and empty. The variant is +**assigned per session** — a review-less browser context stayed review-less +across six consecutive reloads — and a fresh session re-rolls it, at a +measured hit rate of roughly 55%. So the engines start a new browser rather +than reloading, up to five times, which puts the chance of an empty result +under one in fifty. Scrolling does not help and is not tried: six scroll +passes left an empty widget empty. + +Over `--cdp-endpoint` this cannot be fixed, because the remote profile's +cookies are not the client's to discard. Use a different `pid`. + +### Best-seller grids lazy-load + +A grid holds 30 cards on first render and 50 after scrolling to the bottom. +The engines scroll until the count stops growing, and then check the ranks: +because a chart publishes contiguous ranks, 30 rows spanning ranks 1–50 is +arithmetic proof that 20 cards never loaded, and the run says so. + +### The `url` column is rebuilt, not read -## Output fields +A sponsored tile's link is a click tracker with the product path +percent-encoded inside it (`/sspa/click?...&url=%2F...%2Fdp%2FB0C3HCD34R...`), +so reading hrefs both misses sponsored products and stores trackers. Since +the ASIN is on the tile and `https://www.{host}/dp/{ASIN}` always resolves, +the canonical URL is rebuilt from it. That also keeps `url` stable between +runs, which Amazon's `qid`/`xpid`/`ref` parameters are not — otherwise every +row would read as changed on every run. -### Search results -| Field | Type | Description | -|-------|------|-------------| -| `asin` | str | Amazon 10-char identifier | -| `title` | str | Product title | -| `price` | float | Price | -| `currency` | str | Currency (EUR, USD, GBP, JPY…) | -| `rating` | float | Rating 1.0–5.0 | -| `reviews_count` | int | Number of reviews | -| `url` | str | Full product URL | -| `sponsored` | bool | Is sponsored listing | -| `prime_eligible` | bool | Prime delivery available | -| `image_url` | str | Product image URL | +### Best-seller titles are sometimes lower-cased -### Product page -`asin` · `title` · `brand` · `price` · `currency` · `list_price` · `rating` · `reviews_count` · `availability` · `prime_eligible` · `fulfilled_by_amazon` · `seller_name` · `images[]` · `bullets[]` · `description` · `category_breadcrumb[]` · `variations{}` · `deals` · `url` +`blink plus plan with monthly auto-renewal` is what the grid publishes. That +is the site's own text, not a normalisation this scraper applies. -### Best Sellers -`rank` · `asin` · `title` · `price` · `currency` · `rating` · `reviews_count` · `image_url` · `url` · `prime_eligible` +### There is no `prime` column -### Reviews -`review_id` · `author` · `rating` · `title` · `date` · `body` · `verified_purchase` · `helpful_votes` +Amazon rendered no Prime marker on any tile captured from a cross-border exit +(0 of 22 on .com, 0 of 16 on .de, 0 of 60 on .co.jp), so the column would +have been null on every row of every run. A field that looks available and +never is costs more than a missing one. If a domestic exit does render one, +open an issue with the capture and it goes back in. --- -## Anti-bot +## How it parses -| Mechanism | Details | -|-----------|---------| -| Browser fingerprints | Chrome Win/Mac, Firefox, Safari — real UAs + matching sec-ch-ua | -| Accept-Language | Per marketplace: `de-DE` for .de, `ja-JP` for .co.jp, etc. | -| Request delays | Random 2–6s, configurable via `--delay-min` / `--delay-max` | -| 503 / 429 handling | Exponential backoff 8s → 16s → 30s + proxy swap | -| CAPTCHA / block detection | 8 regex patterns, UA profile rotation on block | -| Proxy health tracking | Round-robin, 60s cooldown after 3 bans, drop below 20% success rate | +**Amazon publishes no JSON-LD.** Zero ` + + + +
+ + + +""" + +# Amazon's 503 throttle page from amazon.de — "Tut uns Leid". Retryable: a +# fresh context got 200 seconds later. +FIX_THROTTLE_PAGE_DE = """ + + + + +Tut uns Leid! + + + + + + + + + +
+ +Amazon.de +

+ + + + +
+

Tut uns Leid!

+ + +Während wir Ihre Eingabe ausführen wollten, ist ein technischer Fehler aufgetreten. Wir arbeiten bereits daran und werden sobald wie möglich wieder für Sie da sein. Bitte schauen Sie später wieder vorbei.

+ +Für diese Unannehmlichkeit bitten wir Sie vielmals um Entschuldigung und danken für Ihr Verständnis.

+ +Ihr Team von Amazon.de + +

+Klicken Sie hier, um zurück zur Homepage Amazon.de +

+ + + +""" + + +def page(*fragments): + """Wrap fragments in a minimal document, as the engines hand it over.""" + return "%s" % "".join(fragments) + + +# --------------------------------------------------------------------------- +# Money +# --------------------------------------------------------------------------- +def test_price_parsing(): + group("price and currency parsing") + prices = product_parser._prices_in + ok = True + + ok &= check("'EUR 34.40' (ISO code, NBSP) -> 34.40 EUR", + prices("EUR 34.40") == ([34.40], "EUR")) + # A detail page renders the code hard against the number, a listing tile + # separates them. Both must parse; the earlier pattern required a space + # and read the detail page's price as no price at all. + ok &= check("'EUR19.61' (no separator at all) -> 19.61 EUR", + prices("EUR19.61") == ([19.61], "EUR")) + ok &= check("'179,99 €' (comma decimal, NBSP, trailing symbol) -> 179.99 EUR", + prices("179,99 €") == ([179.99], "EUR")) + ok &= check("'$1,234.56' -> 1234.56 USD", prices("$1,234.56") == ([1234.56], "USD")) + ok &= check("'1.234,56 €' (EU convention) -> 1234.56 EUR", + prices("1.234,56 €") == ([1234.56], "EUR")) + + # Space grouping, in all four space characters a rendered page uses. A + # plain-space-only pattern read "1 234 €" as 234 — an order of magnitude + # out, silently. + for name, space in (("plain space", " "), ("NBSP", " "), + ("narrow NBSP", " "), ("thin space", " ")): + ok &= check("space grouping with %s: '1%s234 €' -> 1234.0" % (name, space), + prices("1%s234 €" % space) == ([1234.0], "EUR")) + + # A single separator followed by exactly three digits is a thousands + # grouping: no currency here has a three-digit subunit. + ok &= check("'$1,234' -> 1234.0, not 1.234", prices("$1,234") == ([1234.0], "USD")) + ok &= check("'$1,23' -> 1.23 (two trailing digits is a decimal)", + prices("$1,23") == ([1.23], "USD")) + + # The ISO-code form is matched against an allowlist. A bare [A-Z]{3} + # would turn a size chart into a price. + ok &= check("'XXL 100' is not a price (XXL is not a currency)", + prices("XXL 100") == ([], None)) + ok &= check("'USB 3' is not a price", prices("USB 3") == ([], None)) + ok &= check("'100 CHF' (trailing ISO code) -> 100.0 CHF", + prices("100 CHF") == ([100.0], "CHF")) + + # Currency tiers. A written code and a compound symbol name themselves; a + # bare local symbol is a guess the marketplace can narrow. + ok &= check("'HK$1,234' -> HKD, not USD (compound symbol beats bare $)", + prices("HK$1,234") == ([1234.0], "HKD")) + ok &= check("bare 'kr' with no marketplace known -> amount, currency None", + prices("199 kr") == ([199.0], None)) + ok &= check("bare 'kr' on amazon.se -> SEK", + prices("199 kr", "SEK") == ([199.0], "SEK")) + ok &= check("bare '$' with amazon.com known -> USD", + prices("$5.00", "USD") == ([5.0], "USD")) + ok &= check("no money in the text -> ([], None), never a defaulted USD", + prices("Free returns") == ([], None)) + return ok + + +# --------------------------------------------------------------------------- +# Listing tiles +# --------------------------------------------------------------------------- +def test_search_tiles(): + group("search tiles (real capture)") + ok = True + rows = parse_products(page(FIX_TILE_SPONSORED_COUPON), + "https://www.amazon.com/s?k=wireless+headphones") + ok &= check("one tile in, one row out", len(rows) == 1) + r = rows[0] + ok &= check("sku is the ASIN from data-asin", r.sku == "B0HGFFLDYL") + # The tile's own link is a /sspa/click tracker with the product path + # percent-encoded inside it, so the row's url is rebuilt from the ASIN: + # stable between runs, and correct for sponsored tiles too. + ok &= check("url is the canonical /dp/{ASIN}, not the sspa click tracker", + r.url == "https://www.amazon.com/dp/B0HGFFLDYL" + and "sspa" not in r.url) + ok &= check("price read from .a-price .a-offscreen (85.15), not the " + "self-concatenated .a-price text (8515)", r.price == 85.15) + ok &= check("currency EUR (a cross-border visitor is served the ISO code)", + r.currency == "EUR") + ok &= check("original_price is the struck-through list price", r.original_price == 170.33) + ok &= check("discount computed from the two prices, not read from a flash", + r.discount_pct == 50.0) + ok &= check("rating 4.6 from the icon's accessible text", r.rating == 4.6) + # 465 is what a digit sweep over "4.6 out of 5 stars, 28 ratings" + # produces. It shipped in live runs before this check existed. + ok &= check("review_count 28 (the number BEFORE 'ratings'), not 465 " + "(rating+scale+count concatenated)", r.review_count == 28) + ok &= check("sponsored flagged, not dropped", r.sponsored is True) + # The label is rendered twice (screen-reader + visible), and the amount + # carries an NBSP that _clean_text normalises to a plain space — so the + # column holds one readable value rather than "Save EUR 42.15 EUR 42.15 + # off coupon" with a character no consumer expects. + ok &= check("coupon label de-duplicated and NBSP-normalised to " + "'Save EUR 42.15'", r.coupon == "Save EUR 42.15") + ok &= check("price_source records which node was read", r.price_source == "offscreen") + ok &= check("position from data-index", r.position == 3) + ok &= check("page defaults to 1 when the URL says nothing", r.page == 1) + ok &= check("category defaults to the search term", r.category == "wireless headphones") + ok &= check("source is the marketplace host", r.source == "amazon.com") + ok &= check("brand is null on a listing tile (Amazon renders no brand " + "field there) rather than guessed from the title", + r.brand is None) + ok &= check("image url captured", (r.image_url or "").startswith("https://m.media-amazon.com/")) + + # The withheld-price tile. Null here is the site's answer, not a failure. + rows = parse_products(page(FIX_TILE_BADGE_COM), + "https://www.amazon.com/s?k=wireless+headphones&page=2") + r = rows[0] + ok &= check("a tile Amazon renders without a price gives price None, and " + "the row is still kept", r.price is None and r.sku == "B0C3HCD34R") + ok &= check("no price means no currency and no price_source either — not " + "a defaulted USD", r.currency is None and r.price_source is None) + ok &= check("badge 'Overall Pick' read", r.badge == "Overall Pick") + ok &= check("not sponsored: False, not None — the page was checked", + r.sponsored is False) + ok &= check("page read from ?page=2, so position is unambiguous across pages", + r.page == 2 and r.position == 4) + ok &= check("review_count 73731 (matches this ASIN's own detail page)", + r.review_count == 73731) + return ok + + +def test_localised_tiles(): + group("localised tiles (amazon.de, amazon.co.jp)") + ok = True + r = parse_products(page(FIX_TILE_DE_SPLIT_BADGE), + "https://www.amazon.de/s?k=kopfhoerer")[0] + ok &= check("de: '15,10 €' parsed as 15.10 EUR", r.price == 15.10 and r.currency == "EUR") + ok &= check("de: list price 19,00 € -> 19.0, discount 20.5%", + r.original_price == 19.0 and r.discount_pct == 20.5) + # Amazon styles a badge by splitting its label across spans. Reading one + # span gives "Amazons". + ok &= check("de: badge joined across spans -> 'Amazons Tipp', not 'Amazons'", + r.badge == "Amazons Tipp") + ok &= check("de: rating 4.6 from 'von 5 Sternen' wording", r.rating == 4.6) + ok &= check("de: source amazon.de and url on that host", + r.source == "amazon.de" and r.url.startswith("https://www.amazon.de/dp/")) + ok &= check("de: zero-width characters stripped from the title", + "​" not in (r.title or "") and r.title == "Apple EarPods (USB-C)") + + r = parse_products(page(FIX_TILE_JP_RATING), + "https://www.amazon.co.jp/s?k=headphones")[0] + # The scale comes FIRST in Japanese. An "N out of 5" pattern matches + # nothing, and every rating on this marketplace was null before. + ok &= check("jp: rating 4.2 from '5つ星のうち 4.2' (scale stated first)", + r.rating == 4.2) + ok &= check("jp: review_count 339 from '339 レーティング', not 542 " + "(scale+rating concatenated)", r.review_count == 339) + ok &= check("jp: price served in EUR to a European exit — currency follows " + "the delivery country, NOT the domain", + r.currency == "EUR" and r.price == 33.61) + ok &= check("jp: source amazon.co.jp", r.source == "amazon.co.jp") + return ok + + +def test_bestseller_cards(): + group("best-seller grid (real capture)") + ok = True + rows = parse_products(page(FIX_BESTSELLER_CARD), + "https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics/") + ok &= check("one card in, one row out", len(rows) == 1) + r = rows[0] + ok &= check("sku from the card's inner data-asin", r.sku == "B08JHCVHTY") + # The class is `_cDEzb_p13n-sc-price_3mJ9Z` — a build hash. Matching the + # whole name would break on Amazon's next deploy. + ok &= check("price read through a substring class match, despite the " + "build-hashed class name", r.price == 10.31) + ok &= check("position is the PUBLISHED rank (#1), read not counted", + r.position == 1) + ok &= check("review_count 279961, not 445279961 (the digit sweep that " + "shipped garbage in a live run)", r.review_count == 279961) + ok &= check("rating 4.4", r.rating == 4.4) + ok &= check("category from the department slug", r.category == "electronics") + ok &= check("a grid shows one price, so original_price stays null", + r.original_price is None) + return ok + + +def test_url_fallback_and_tile_scope(): + group("URL-pattern fallback and tile scoping") + ok = True + # Neither primary anchor present: the parser must still return rows + # rather than report an empty category. + fallback_html = """ +
+
+ First product + EUR 10.00 +
+
+ Second product + EUR 99.00 +
+
""" + rows = parse_products(page(fallback_html), "https://www.amazon.com/s?k=x") + ok &= check("fallback path finds both products by URL pattern", len(rows) == 2) + by_sku = {r.sku: r for r in rows} + # One level too wide and every tile reports its neighbours' prices — the + # "junk-link data theft" failure this scoping exists to prevent. + ok &= check("each row gets its OWN tile's price, not its neighbour's", + by_sku["B000000001"].price == 10.0 + and by_sku["B000000002"].price == 99.0) + ok &= check("fallback rows say where the price came from", + all(r.price_source == "split" for r in rows)) + + # A sponsored click-tracker hides the real path inside a query parameter. + spa = ("/sspa/click?ie=UTF8&spc=abc&url=%2FCancelling-Headphones%2Fdp%2F" + "B0C3HCD34R%2Fref%3Dsr_1_3&qualifier=1") + ok &= check("ASIN recovered from a percent-encoded sspa click tracker", + product_parser._asin_from_href(spa) == "B0C3HCD34R") + ok &= check("ASIN recovered from a plain /dp/ href", + product_parser._asin_from_href("/x/dp/B000000003/ref=y") == "B000000003") + ok &= check("a href with no ASIN gives None", + product_parser._asin_from_href("/gp/help/customer/display.html") is None) + + # data-asin="" appears on spacer rows inside the results list. + ok &= check("an empty data-asin is not an ASIN", + product_parser._valid_asin("") is None + and product_parser._valid_asin(None) is None) + ok &= check("a 9-character id is not an ASIN", + product_parser._valid_asin("B00000000") is None) + return ok + + +# --------------------------------------------------------------------------- +# Detail page and reviews +# --------------------------------------------------------------------------- +DETAIL_PAGE = None # built in main(), from the fragments below + + +def test_product_detail(): + group("product detail page (real capture)") + ok = True + html = page(FIX_DETAIL_TITLE, FIX_DETAIL_BYLINE, FIX_DETAIL_PRICE_DESKTOP, + FIX_DETAIL_PRICE_CORE, FIX_DETAIL_ACR, FIX_DETAIL_AVAILABILITY, + FIX_DETAIL_BREADCRUMBS, FIX_DETAIL_BULLETS) + rows = parse_product_detail(html, "https://www.amazon.com/dp/B07K5214NZ") + ok &= check("a detail page yields exactly one row", len(rows) == 1) + r = rows[0] + ok &= check("sku from the URL", r.sku == "B07K5214NZ") + # The desktop price container is PRESENT AND EMPTY on this capture while + # the next one holds the number, and the list price lives in the same + # container as the price. Both are why the price used to come back as + # 20.64 — the struck-through figure. + ok &= check("price is 19.61 (what is charged), not 20.64 (struck through)", + r.price == 19.61) + ok &= check("original_price is the struck-through 20.64", r.original_price == 20.64) + ok &= check("discount 5.0% computed from the pair", r.discount_pct == 5.0) + ok &= check("price_source says 'detail'", r.price_source == "detail") + ok &= check("brand unwrapped from 'Visit the ZIHNIC Store'", r.brand == "ZIHNIC") + # Read from the whole document, this matched a carousel item's aria-label + # and reported ITS count (13,930) for this product. + ok &= check("review_count 87349 from this product's own block, not a " + "carousel item's", r.review_count == 87349) + ok &= check("rating 4.4", r.rating == 4.4) + ok &= check("category from the site's own breadcrumb", + (r.category or "").startswith("Electronics > Headphones")) + ok &= check("availability text kept verbatim", r.availability == "In Stock") + ok &= check("bullets captured as a list", len(r.bullets or []) == 5) + ok &= check("listing-only columns stay null on a detail row", + r.position is None and r.sponsored is None and r.page is None) + + # in_stock is decided from MARKUP, not from words: #availability reads + # "In Stock" / "Auf Lager" / "在庫あり" depending on the marketplace. + ok &= check("in_stock is None when neither control is present — 'we could " + "not tell' is not 'no'", r.in_stock is None) + with_cart = parse_product_detail( + page(FIX_DETAIL_TITLE, ''), + "https://www.amazon.com/dp/B07K5214NZ") + ok &= check("in_stock True from an add-to-cart control, language-neutrally", + with_cart and with_cart[0].in_stock is True) + out_of_stock = parse_product_detail( + page(FIX_DETAIL_TITLE, '
Derzeit nicht verfügbar
'), + "https://www.amazon.com/dp/B07K5214NZ") + ok &= check("in_stock False from the out-of-stock block", + out_of_stock and out_of_stock[0].in_stock is False) + + # A page that is not a detail page must give nothing rather than a row of + # nulls that looks like a product. + ok &= check("no #productTitle -> no row (a warning, not a fake product)", + parse_product_detail(page("
nothing here
"), + "https://www.amazon.com/dp/B07K5214NZ") == []) + return ok + + +def test_reviews(): + group("reviews rendered on a detail page (real capture)") + ok = True + rows = parse_reviews(page(FIX_REVIEW_CONTAINER), + "https://www.amazon.com/dp/B07K5214NZ") + ok &= check("one container in, one Review out", len(rows) == 1) + r = rows[0] + ok &= check("review_id from data-reviewid", r.review_id == "RXN5D0N6KQ6KP") + ok &= check("rating 5.0 from the review's own star node", r.rating == 5.0) + ok &= check("author read", r.author == "Angela") + ok &= check("verified purchase flagged", r.verified_purchase is True) + ok &= check("helpful votes parsed from '4 people found this helpful'", + r.helpful_votes == 4) + ok &= check("variant captured", r.variant == "Color: Rose") + ok &= check("date kept verbatim, not parsed to a date", + (r.review_date or "").startswith("Reviewed in the United States on")) + # The body node's own text begins with an accessibility affordance. + ok &= check("accessibility filler stripped from the body", + "double tap" not in (r.body or "") + and (r.body or "").startswith("Very comfortable headphones!")) + ok &= check("title has no rating text prefixed", + r.title == "Great sound and very comfortable") + ok &= check("url is the per-review permalink, not the product page", + r.url == "https://www.amazon.com/gp/customer-reviews/RXN5D0N6KQ6KP") + ok &= check("sku is the ASIN that was REQUESTED", r.sku == "B07K5214NZ") + ok &= check("variant_asin null when the review is on the same ASIN", + r.variant_asin is None) + + # A review attached to a colour variant: the row must still be joinable + # to what the caller asked for. + variant_html = FIX_REVIEW_CONTAINER.replace('data-asin="B07K5214NZ"', + 'data-asin="B0CQXMXJC5"', 1) + v = parse_reviews(page(variant_html), "https://www.amazon.com/dp/B07K5214NZ")[0] + ok &= check("a variant's review keys on the requested ASIN, with the " + "variant recorded separately", + v.sku == "B07K5214NZ" and v.variant_asin == "B0CQXMXJC5") + + # "One person found this helpful" has no digit in it at all. + worded = FIX_REVIEW_CONTAINER.replace("4 people found this helpful", + "One person found this helpful") + w = parse_reviews(page(worded), "https://www.amazon.com/dp/B07K5214NZ")[0] + ok &= check("'One person found this helpful' -> 1, not None", + w.helpful_votes == 1) + return ok + + +# --------------------------------------------------------------------------- +# URLs, marketplaces, page state +# --------------------------------------------------------------------------- +def test_urls(): + group("pagination, categories and page kinds") + ok = True + ok &= check("search paginates with ?page=N", + page_url("https://www.amazon.com/s?k=x", 3) + == "https://www.amazon.com/s?k=x&page=3") + # The two listing kinds use DIFFERENT parameters. Using one for the other + # silently re-fetches page 1. + ok &= check("a best-seller grid paginates with ?pg=N, not ?page=N", + page_url("https://www.amazon.com/zgbs/electronics/", 3) + == "https://www.amazon.com/zgbs/electronics/?pg=3") + ok &= check("existing filters are preserved", + "rh=n%3A123" in page_url("https://www.amazon.com/s?k=x&rh=n%3A123", 2)) + ok &= check("an existing page parameter is REPLACED, not appended twice", + page_url("https://www.amazon.com/s?k=x&page=7", 2).count("page=") == 1) + ok &= check("an existing pg parameter is replaced on a grid", + page_url("https://www.amazon.com/zgbs/e/?pg=1", 4).count("pg=") == 1) + + ok &= check("page number read back from a search URL", + page_number_from_url("https://www.amazon.com/s?k=x&page=5") == 5) + ok &= check("page number read back from a grid URL", + page_number_from_url("https://www.amazon.com/zgbs/e/?pg=2") == 2) + ok &= check("no page parameter means page 1", + page_number_from_url("https://www.amazon.com/s?k=x") == 1) + + for url, kind in (("https://www.amazon.com/s?k=x", "search"), + ("https://www.amazon.com/Best-Sellers/zgbs/electronics/", "bestsellers"), + ("https://www.amazon.com/gp/bestsellers/electronics/", "bestsellers"), + ("https://www.amazon.com/dp/B07K5214NZ", "detail"), + ("https://www.amazon.com/gp/product/B07K5214NZ", "detail"), + ("https://www.amazon.com/", "other")): + ok &= check("listing_kind(%s) == %s" % (url[24:] or "/", kind), + listing_kind(url) == kind) + + ok &= check("category from a search URL is the query term", + category_from_url("https://www.amazon.com/s?k=wireless+headphones") + == "wireless headphones") + ok &= check("category from a grid URL is the department slug", + category_from_url("https://www.amazon.com/Best-Sellers-Electronics/zgbs/electronics/") + == "electronics") + ok &= check("a node id in the path is not a category", + category_from_url("https://www.amazon.co.jp/zgbs/electronics/2151981051/") + == "electronics") + # Returning the ASIN made every detail row's category its own id. + ok &= check("a /dp/ URL has no category (the ASIN is not one)", + category_from_url("https://www.amazon.com/dp/B07K5214NZ") is None) + return ok + + +def test_marketplaces(): + group("marketplaces") + ok = True + ok &= check("21 marketplaces registered", len(MARKETPLACES) == 21) + ok &= check("www. stripped so one marketplace is one value", + marketplace_host("https://www.amazon.co.uk/s?k=x") == "amazon.co.uk") + ok &= check("smile. stripped too", + marketplace_host("https://smile.amazon.com/s?k=x") == "amazon.com") + ok &= check("marketplace currency available for symbol disambiguation", + marketplace_info("https://www.amazon.se/s?k=x")["currency"] == "SEK") + # Amazon opens marketplaces faster than a table gets updated. + ok &= check("an unknown amazon host still parses, just without the " + "currency hint", + marketplace_host("https://www.amazon.co.za/s?k=x") == "amazon.co.za" + and marketplace_info("https://www.amazon.co.za/s?k=x") == {}) + ok &= check("every registry entry has country, currency and name", + all({"country", "currency", "name"} <= set(v) + for v in MARKETPLACES.values())) + return ok + + +def test_page_state(): + group("page-state triage (four answers, four different responses)") + ok = True + # The AWS WAF interstitial clears ITSELF in a browser. Reporting it as a + # block would report a block that does not exist; sending it to a solver + # would bill for a challenge no solver can answer. + ok &= check("AWS WAF challenge page detected as waf_challenge", + detect_page_state(FIX_WAF_CHALLENGE_PAGE) == "waf_challenge") + ok &= check("the WAF challenge is NOT reported as a block", + detect_bot_challenge(FIX_WAF_CHALLENGE_PAGE) is None) + ok &= check("Amazon's 503 page detected as throttled", + detect_page_state(FIX_THROTTLE_PAGE_DE) == "throttled") + ok &= check("a bare 503 status with no marker is still throttled", + detect_page_state("nothing", status=503) == "throttled") + ok &= check("the image captcha page detected as captcha", + detect_page_state(CAPTCHA_PAGE) == "captcha") + ok &= check("the image captcha IS reported as a block (exit 3 territory)", + detect_bot_challenge(CAPTCHA_PAGE) == "amazon-captcha") + # Decided from the URL: the real signin page contains "/ap/signin" zero + # times, while every ordinary Amazon page links to it from the header. + ok &= check("a signin redirect is read from the URL", + detect_page_state("irrelevant", + url="https://www.amazon.com/ap/signin?openid.x=1") + == "signin") + ok &= check("a healthy search page is 'ok'", + detect_page_state(page(FIX_TILE_SPONSORED_COUPON), + url="https://www.amazon.com/s?k=x") == "ok") + # This is the regression: a live run reported "needs sign-in" for every + # best-seller grid, because the header's signin link was being matched. + ok &= check("a page merely LINKING to /ap/signin is not a signin page", + detect_page_state('Sign in' + '
', + url="https://www.amazon.com/zgbs/electronics/") + == "ok") + return ok + + +# --------------------------------------------------------------------------- +# The one fixture that is NOT a capture +# --------------------------------------------------------------------------- +# Amazon's image captcha, written from its documented markup rather than +# captured: no run during development was served one. 47 navigations across +# five marketplaces produced AWS WAF challenges (which clear themselves) and +# 503 throttles, and no image captcha at all. Said out loud here because the +# rest of this file's fixtures are real, and a reader is entitled to know +# which one is not — the solve path it exercises is therefore unproven +# against the live site, and TROUBLESHOOTING.md says so too. +CAPTCHA_PAGE = """Amazon.com +
+ + +
+

Enter the characters you see below

+

Sorry, we just need to make sure you're not a robot.

+ + +
+ +
""" + + +# --------------------------------------------------------------------------- +# The output contract +# --------------------------------------------------------------------------- +def test_output_contract(): + group("output contract") + ok = True + # The first sixteen columns are the family's, in the family's order, so a + # consumer written against another repo in this family still reads them. + family_prefix = ["source", "scraped_at", "url", "sku", "title", "brand", + "price", "currency", "original_price", "discount_pct", + "rating", "review_count", "in_stock", "image_url", + "category", "price_source"] + names = [f.name for f in fields(Product)] + ok &= check("Product keeps the family's 16 columns first, in order", + names[:16] == family_prefix) + ok &= check("Amazon's own columns come after them", + names[16:] == ["page", "position", "sponsored", "badge", "coupon", + "seller", "availability", "bullets", "images", + "variations"]) + # A column that is null on every row of every run is worse than a missing + # one; Amazon rendered no Prime marker on any captured tile. + ok &= check("there is no 'prime' column", "prime" not in names) + ok &= check("Review is its own schema, keyed on sku like the others", + [f.name for f in fields(Review)][:5] + == ["source", "scraped_at", "url", "sku", "review_id"]) + ok &= check("currency has no default", Product().currency is None) + ok &= check("modes map to row classes", + ROW_CLASS_BY_MODE == {"listing": Product, "product": Product, + "reviews": Review}) + ok &= check("exit codes are the family's: 3 blocked, 4 empty, 6 partial", + (EXIT_BLOCKED, EXIT_NO_PRODUCTS, EXIT_PARTIAL) == (3, 4, 6)) + ok &= check("single_page_mode counts as a complete run", + "single_page_mode" in COMPLETE_STOP_REASONS) + return ok + + +def test_writers(): + group("writers, dedupe and exit codes") + ok = True + tmp = tempfile.mkdtemp() + + # A run that finds nothing must not replace last night's good output. + out = os.path.join(tmp, "empty") + rc = save([], out, "both") + ok &= check("0 rows: nothing written, exit 4", + rc == EXIT_NO_PRODUCTS + and not os.path.exists(out + ".json") + and not os.path.exists(out + ".csv")) + rc = save([], out, "both", allow_empty=True) + ok &= check("--allow-empty writes the files, and the code is still 4", + rc == EXIT_NO_PRODUCTS and os.path.exists(out + ".json")) + # A zero-byte CSV makes a consumer fail on read instead of reading a + # valid table with no rows. + header = open(out + ".csv", encoding="utf-8").read().strip() + ok &= check("an empty CSV still carries its header", + header.startswith("source,scraped_at,url,sku,title")) + + reviews_out = os.path.join(tmp, "empty_reviews") + write_csv([], reviews_out + ".csv", row_cls=Review) + ok &= check("an empty reviews CSV carries REVIEWS columns, not product ones", + open(reviews_out + ".csv", encoding="utf-8").read().startswith( + "source,scraped_at,url,sku,review_id")) + + # CSV cannot hold a list; repr() of one is neither readable nor parseable. + listed = os.path.join(tmp, "listed") + save([Product(sku="A", bullets=["one", "two"])], listed, "csv") + row = open(listed + ".csv", encoding="utf-8").read().splitlines()[1] + ok &= check("a list column is joined with '%s' in CSV, not repr()ed" + % LIST_CSV_SEPARATOR.strip(), + "one | two" in row and "['one'" not in row) + + seen = set() + dupes = [Product(sku="A"), Product(sku="A"), Product(sku="B"), Product(sku=None)] + kept = dedupe_by_sku(dupes, seen) + ok &= check("dedupe drops a repeated sku and KEEPS a row with no sku " + "(nothing to key on is not a duplicate)", len(kept) == 3) + seen = set() + revs = [Review(sku="A", review_id="r1"), Review(sku="A", review_id="r1"), + Review(sku="A", review_id="r2")] + ok &= check("reviews dedupe on review_id, so a dozen reviews of one ASIN " + "all survive", len(dedupe_by_key(revs, seen, "review_id")) == 2) + + meta = run_meta("complete", "completed", 3, 3, "u", "u2", 5, mode="reviews", + source="amazon.de") + ok &= check("the sidecar records mode and source — neither is implied by " + "the repo", meta["mode"] == "reviews" and meta["source"] == "amazon.de") + ok &= check("the sidecar names WHICH pages failed, not just how many", + "pages_failed" in meta) + + # status/exit mapping, shared so the three engines cannot drift + cases = [ + (dict(blocked=True, stop_reason="blocked_amazon-captcha", rows=[]), EXIT_BLOCKED, None), + (dict(blocked=False, stop_reason="completed", rows=[]), EXIT_NO_PRODUCTS, None), + (dict(blocked=False, stop_reason="completed", rows=[Product(sku="A")]), 0, "complete"), + (dict(blocked=False, stop_reason="no_new_products", rows=[Product(sku="A")]), 0, "complete"), + (dict(blocked=False, stop_reason="single_page_mode", rows=[Product(sku="A")]), 0, "complete"), + (dict(blocked=False, stop_reason="page_load_timeout", rows=[Product(sku="A")]), EXIT_PARTIAL, "partial"), + ] + for i, (kw, expected_rc, expected_status) in enumerate(cases): + prefix = os.path.join(tmp, "run%d" % i) + rc = finish_run(kw["rows"], prefix, "json", False, blocked=kw["blocked"], + stop_reason=kw["stop_reason"], pages_requested=1, + pages_completed=1, start_url="u", final_url="u") + got_status = None + if os.path.exists(prefix + ".meta.json"): + got_status = json.load(open(prefix + ".meta.json"))["status"] + ok &= check("finish_run(%s, %d row) -> exit %d, status %s" + % (kw["stop_reason"], len(kw["rows"]), expected_rc, expected_status), + rc == expected_rc and got_status == expected_status) + # A "failed" sidecar beside the previous run's still-good output would + # contradict it, and diff_runs would refuse data that is fine. + ok &= check("a failed run writes NO sidecar", + not os.path.exists(os.path.join(tmp, "run0.meta.json"))) + return ok + + +def test_diff(): + group("diff_runs") + ok = True + old = [{"sku": "A", "title": "a", "price": 25.80, "original_price": 34.40, + "discount_pct": 25.0, "currency": "EUR", "in_stock": None, + "price_source": "offscreen"}, + {"sku": "B", "title": "b", "price": 100.0, "currency": "EUR", + "price_source": "offscreen"}, + {"sku": "C", "title": "c", "price": 5.0, "currency": "EUR", + "price_source": "offscreen"}] + new = [{"sku": "A", "title": "a", "price": 25.81, "original_price": 34.42, + "discount_pct": 25.0, "currency": "EUR", "in_stock": None, + "price_source": "offscreen"}, + {"sku": "B", "title": "b", "price": 89.0, "currency": "EUR", + "price_source": "offscreen"}, + {"sku": "D", "title": "d", "price": 7.0, "currency": "EUR", + "price_source": "offscreen"}] + + r = diff_products(old, new) + ok &= check("added/removed keyed on sku", + [p["sku"] for p in r["added"]] == ["D"] + and [p["sku"] for p in r["removed"]] == ["C"]) + ok &= check("with the default tolerance of 0, every cent is reported", + len(r["changed"]) == 2 and not r["within_tolerance"]) + + # Two runs of the same command ten minutes apart disagreed on 10 of 43 + # ASINs, all by 0.039-0.058%: Amazon converts the price for a + # cross-border visitor and the exchange rate ticks. + r = diff_products(old, new, price_tolerance_pct=0.1) + ok &= check("a 0.04% move inside --price-tolerance-pct is an FX tick, not " + "a change", [c["sku"] for c in r["within_tolerance"]] == ["A"]) + ok &= check("an 11% cut is still a change at that tolerance", + [c["sku"] for c in r["changed"]] == ["B"]) + + # A currency change of any size is a real change. + r = diff_products( + [{"sku": "A", "price": 5.0, "currency": "EUR", "price_source": "offscreen"}], + [{"sku": "A", "price": 5.0, "currency": "USD", "price_source": "offscreen"}], + price_tolerance_pct=99.0) + ok &= check("a currency change is never swallowed by the tolerance", + len(r["changed"]) == 1) + + # A differing price_source means our two snapshots read different nodes. + r = diff_products( + [{"sku": "A", "price": 5.0, "currency": "EUR", "price_source": "offscreen"}], + [{"sku": "A", "price": 6.0, "currency": "EUR", "price_source": "split"}]) + ok &= check("a price change with a price_source change is reported as " + "source_changed, not changed", + len(r["source_changed"]) == 1 and not r["changed"]) + return ok + + +def test_captcha(): + group("captcha detection") + ok = True + c = detect_amazon_captcha(CAPTCHA_PAGE, "https://www.amazon.com/s?k=x") + ok &= check("Amazon's image captcha detected", c is not None) + ok &= check("the captcha image URL is captured", + c.image_url.endswith("Captcha_qhlmauzbfe.jpg")) + # Both hidden fields must be echoed back: dropping amzn-r lands the + # browser on the home page having spent a solve. + ok &= check("both hidden form fields captured", + c.amzn == "AbCdEf+/123=" and c.amzn_r == "/s?k=wireless+headphones") + submit = amazon_captcha_submit_url(c, "HELLO") + ok &= check("the solution is submitted as a GET with all three parameters", + "field-keywords=HELLO" in submit and "amzn=" in submit + and "amzn-r=" in submit + and submit.startswith("https://www.amazon.com/errors/validateCaptcha")) + # "/errors/validateCaptcha" appears in Amazon's own error-handling + # JavaScript on ordinary pages, so a marker alone must not fire — under + # --solve-captcha always that would buy a solve per page. + ok &= check("a marker with no captcha image is NOT a captcha", + detect_amazon_captcha( + '', + "https://www.amazon.com/s?k=x") is None) + ok &= check("a healthy listing page is not a captcha", + detect_amazon_captcha(page(FIX_TILE_SPONSORED_COUPON), + "https://www.amazon.com/s?k=x") is None) + # reCAPTCHA detection stays in place: Amazon uses it on account flows, + # and the family's rule is that detection stays broad. + ok &= check("the reCAPTCHA v3 detector still fires on an inline " + "grecaptcha.execute call", + detect_recaptcha_v3( + "", + "https://www.amazon.com/ap/signin") is not None) + # A pinned LIMITATION rather than a half-guard: a bare g-recaptcha + # container with the config supplied in JavaScript is invisible to the + # static detector, which is precisely why the runtime detector exists and + # why both are run and reconciled. Asserting the current behaviour makes a + # future change to it a decision rather than a surprise. + ok &= check("a bare g-recaptcha div with no sitekey in the HTML is NOT " + "detected statically — that is what the runtime detector is for", + detect_recaptcha_v3( + '
', + "https://www.amazon.com/ap/signin") is None) + ok &= check("detect_recaptcha_in_page is present for the runtime path", + callable(detect_recaptcha_in_page)) + ok &= check("reconcile_detections still returns None for a clean page", + reconcile_detections(None, None) is None) + return ok + + +# --------------------------------------------------------------------------- +# The shared policy +# --------------------------------------------------------------------------- +class FakeDriver: + """A scriptable stand-in for a browser, for page_flow's algorithms. + + `counts` is what count() returns on each successive call; the last value + repeats. Records what it was asked to do so the tests can assert the + ORDER of operations, which is where two real bugs lived. + """ + + def __init__(self, counts, heights=None, html_sequence=None): + self.counts = list(counts) + self.heights = list(heights or [1000]) + self.html_sequence = list(html_sequence or []) + self.calls = [] + + def _next(self, seq): + return seq.pop(0) if len(seq) > 1 else (seq[0] if seq else 0) + + def count(self, selector): + self.calls.append(("count", selector)) + return self._next(self.counts) + + def page_height(self): + return self._next(self.heights) + + def scroll_to_bottom(self): + self.calls.append(("scroll_to_bottom",)) + + def scroll_into_view(self, selector): + self.calls.append(("scroll_into_view", selector)) + + def sleep(self, ms): + self.calls.append(("sleep", ms)) + + def content(self): + return self._next(self.html_sequence) if self.html_sequence else "" + + def current_url(self): + return "https://www.amazon.com/s?k=x" + + +def test_page_flow(): + group("page_flow policy") + ok = True + ok &= check("a listing needs several anchors before it counts as loaded", + page_flow.min_matches("listing") == page_flow.MIN_CARD_MATCHES + and page_flow.MIN_CARD_MATCHES > 1) + # A detail page has exactly one #productTitle: requiring more than one + # would time out on every successful fetch. + ok &= check("a detail page needs only one anchor (the threshold follows " + "the mode, or it silently inverts)", + page_flow.min_matches("product") == 0 + and page_flow.min_matches("reviews") == 0) + ok &= check("the listing readiness selector is the PRIMARY anchor, not " + "a[href*='/dp/'] (143 such links for 22 tiles)", + "s-search-result" in page_flow.READY_SELECTOR_LISTING + and "p13n-asin-index" in page_flow.READY_SELECTOR_LISTING + and "/dp/" not in page_flow.READY_SELECTOR_LISTING) + ok &= check("link[rel=next] leads the pagination selector even though " + "Amazon serves none — the durable form comes first", + page_flow.NEXT_PAGE_SELECTOR.startswith("link[rel='next']")) + # Search results are server-rendered: a captured page held the same 16 + # tiles before and after scrolling to the bottom. + ok &= check("search results are not scrolled (they do not grow)", + page_flow.needs_scrolling("listing", "https://www.amazon.com/s?k=x") is False) + ok &= check("best-seller grids ARE scrolled (30 -> 50 cards, measured)", + page_flow.needs_scrolling("listing", "https://www.amazon.com/zgbs/e/") is True) + ok &= check("detail pages are scrolled", + page_flow.needs_scrolling("product", "https://www.amazon.com/dp/B0000000AA") is True) + ok &= check("a detail page gets a shorter content timeout than a listing", + page_flow.content_timeout_ms("reviews") < page_flow.content_timeout_ms("listing")) + # Reviews fail because of a served variant, not a transient fault, so + # they get their own session budget rather than eating --retries. + ok &= check("reviews get their own session budget (>=5), independent of " + "--retries", page_flow.session_attempts("reviews", 1) >= 5 + and page_flow.session_attempts("listing", 1) == 1) + ok &= check("scrolling is not retried for reviews — measured not to be " + "what fills the empty variant", + page_flow.hydrate_attempts("reviews") == 1 + and page_flow.hydrate_attempts("listing") == 3) + + # The count must hold still for several rounds: one quiet round is not + # the end of the content, and believing it cost 20 of 50 cards. + d = FakeDriver(counts=[30, 30, 30, 50, 50, 50, 50], heights=[1, 1, 1, 2, 2, 2, 2]) + got = page_flow.scroll_until_stable(d.count, d.page_height, d.scroll_to_bottom, + d.sleep, "sel", stable_rounds=3) + ok &= check("scroll_until_stable waits out a plateau and returns the " + "final count (50, not 30)", got == 50) + ok &= check("it scrolls to the document end rather than wheeling a fixed " + "distance", ("scroll_to_bottom",) in d.calls) + d = FakeDriver(counts=[7], heights=[1]) + ok &= check("a page that never grows stops after stable_rounds", + page_flow.scroll_until_stable(d.count, d.page_height, + d.scroll_to_bottom, d.sleep, "sel") == 7) + d = FakeDriver(counts=list(range(1, 200)), heights=list(range(1, 200))) + ok &= check("a page that grows forever is bounded by max_rounds", + page_flow.scroll_until_stable(d.count, d.page_height, + d.scroll_to_bottom, d.sleep, "sel", + max_rounds=4) is not None) + + # The review widget hydrates from an observer on its own element, so the + # anchor is scrolled into view before the bottom-ward pass. + d = FakeDriver(counts=[0, 0, 13, 13, 13, 13]) + page_flow.hydrate(d.count, d.page_height, d.scroll_to_bottom, d.sleep, + d.scroll_into_view, "reviews", "sel", 0, attempts=2) + scrolled_anchor = [c for c in d.calls if c[0] == "scroll_into_view"] + ok &= check("hydrate scrolls the mode's own anchor into view first", + scrolled_anchor + and "reviewsMedley" in scrolled_anchor[0][1]) + + # The WAF challenge must be waited out, not reported. + d = FakeDriver(counts=[0], html_sequence=[FIX_WAF_CHALLENGE_PAGE, + FIX_WAF_CHALLENGE_PAGE, + page(FIX_TILE_SPONSORED_COUPON)]) + ok &= check("wait_out_waf_challenge returns True once the real page arrives", + page_flow.wait_out_waf_challenge(d.content, d.current_url, d.sleep) is True) + d = FakeDriver(counts=[0], html_sequence=[FIX_WAF_CHALLENGE_PAGE]) + ok &= check("a challenge that never clears gives up (bounded) and says so", + page_flow.wait_out_waf_challenge(d.content, d.current_url, d.sleep) is False) + return ok + + +# --------------------------------------------------------------------------- +# Configuration and credentials +# --------------------------------------------------------------------------- +class _Args: + def __init__(self, **kw): + self.__dict__.update(kw) + + +def test_env_config(): + group("configuration") + ok = True + ok &= check("ENV_KEYS names the Amazon variables", + set(env_config.ENV_KEYS) == {"TWOCAPTCHA_KEY", "AMAZON_CDP_ENDPOINT", + "AMAZON_PROXY", "AMAZON_URL"}) + # A variable mapped onto a flag with a non-empty default would be + # silently inert: a setting that looks configurable and is not. + ok &= check("no variable maps onto --out, which has a non-empty default", + "out" not in env_config.ENV_KEYS.values()) + + # .env.example must document exactly what the code reads, in both + # directions — it drifts otherwise. + example = os.path.join(REPO_ROOT, ".env.example") + documented = set() + with open(example, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + documented.add(line.split("=", 1)[0].strip()) + ok &= check(".env.example documents exactly the variables the code reads", + documented == set(env_config.ENV_KEYS)) + + # An explicit flag always wins over the environment. + saved = os.environ.get("AMAZON_URL") + os.environ["AMAZON_URL"] = "https://www.amazon.com/s?k=from-env" + try: + args = _Args(url="https://www.amazon.com/s?k=from-flag", twocaptcha_key=None, + cdp_endpoint=None, proxy=None) + env_config.apply(args) + ok &= check("an explicit flag beats an environment variable", + args.url == "https://www.amazon.com/s?k=from-flag") + args = _Args(url=None, twocaptcha_key=None, cdp_endpoint=None, proxy=None) + env_config.apply(args) + ok &= check("an unset flag is filled from the environment", + args.url == "https://www.amazon.com/s?k=from-env") + finally: + if saved is None: + del os.environ["AMAZON_URL"] + else: + os.environ["AMAZON_URL"] = saved + return ok + + +def test_proxy_pool(): + group("proxies and credentials") + ok = True + url = "http://user:secret@eu.proxy.2captcha.com:2334" + # Host and port are the point of a rotation log and are not the secret. + ok &= check("mask() hides credentials and KEEPS host and port", + mask(url) == "http://***:***@eu.proxy.2captcha.com:2334") + ok &= check("the password never appears in a masked URL", "secret" not in mask(url)) + # --proxy-server= becomes part of the browser's argv, readable by + # anything that can run `ps`. + pw = to_playwright(url) + ok &= check("Playwright gets credentials in their own fields, not in server", + pw["server"] == "http://eu.proxy.2captcha.com:2334" + and pw["username"] == "user" and pw["password"] == "secret" + and "secret" not in pw["server"]) + scrubbed, creds = split_credentials(url) + ok &= check("split_credentials keeps the address argv-safe", + scrubbed == "http://eu.proxy.2captcha.com:2334" + and creds == ("user", "secret")) + ok &= check("a proxy with no credentials yields None for them", + split_credentials("http://host:1234")[1] is None) + + pool = ProxyPool(["http://a:1", "http://b:2", "http://c:3"], rotate="per-page") + first = pool.current + pool.advance("test") + ok &= check("advance() moves to a different exit", pool.current != first) + ok &= check("per-page rotation is reported by rotates_per_page()", + pool.rotates_per_page() is True) + ok &= check("per-run rotation is not", + ProxyPool(["http://a:1"], rotate="per-run").rotates_per_page() is False) + ok &= check("a comment line in a proxy file is skipped", + parse_proxy_line("# a comment") is None) + return ok + + +# --------------------------------------------------------------------------- +# Engines +# --------------------------------------------------------------------------- +ENGINES = ("playwright_scraper", "puppeteer_scraper", "selenium_scraper") + + +def test_engines(skips): + group("engines (all three must agree)") + ok = True + loaded = {} + for name in ENGINES: + try: + loaded[name] = __import__(name) + except ImportError as e: + # Visible, and CI's engine-smoke job fails on it: "skipped, + # engine absent" reads identically to a real import error. + skips.append("%s (%s)" % (name, e)) + print(" SKIP %s could not be imported: %s" % (name, e)) + + if not loaded: + return ok + + # The public entry point, not only the helpers underneath it: a signature + # once drifted from its callers while every check exercised the privates. + for name, mod in loaded.items(): + ok &= check("%s exposes scrape() and parse_args()" % name, + callable(getattr(mod, "scrape", None)) + and callable(getattr(mod, "parse_args", None))) + ok &= check("%s builds its rows through the shared finish_run()" % name, + "finish_run" in inspect.getsource(mod.scrape)) + ok &= check("%s takes its page policy from page_flow, not its own copy" + % name, "page_flow" in inspect.getsource(mod)) + source = inspect.getsource(mod) + # The AWS WAF interstitial must never reach the captcha solver. + ok &= check("%s waits out the WAF challenge instead of solving it" % name, + "wait_out_waf_challenge" in source) + ok &= check("%s retries Amazon's 503 throttle page" % name, + "throttled" in source) + + # THE GUARD FOR THE GUARD. Everything above only means something if + # importing an engine module genuinely requires its driver library: the + # suite reports a skip when the import fails, and CI's engine-smoke job + # fails on any reported skip. This engine's pyppeteer import once sat + # inside the launch path instead, so the module imported cleanly with no + # pyppeteer installed, the group never skipped, and CI happily ran against + # a stub version (pyppeteer 0.0.25, resolved from an unpinned install) + # without noticing. Checked on the SOURCE rather than by importing, + # because by the time this runs the library is already loaded. + DRIVER_MODULES = {"playwright_scraper": "playwright", + "puppeteer_scraper": "pyppeteer", + "selenium_scraper": "selenium"} + for name, mod in loaded.items(): + driver = DRIVER_MODULES[name] + tree = ast.parse(inspect.getsource(mod)) + top_level = any( + (isinstance(node, ast.ImportFrom) and (node.module or "").split(".")[0] == driver) + or (isinstance(node, ast.Import) + and any(a.name.split(".")[0] == driver for a in node.names)) + for node in tree.body) + ok &= check("%s imports %s at module level, so a missing driver is a " + "reported skip rather than a silently vacuous check" + % (name, driver), top_level) + + # Flag parity: the family's contract is that the three CLIs take the same + # flags, so a script can switch engines without rewriting its arguments. + flag_sets = {} + for name, mod in loaded.items(): + parser = mod.parse_args.__wrapped__ if hasattr(mod.parse_args, "__wrapped__") else None + # parse_args() reads sys.argv, so the flags are read out of the + # source rather than by building the parser. + flag_sets[name] = set(re.findall(r'add_argument\("(--[a-z-]+)"', + inspect.getsource(mod.parse_args))) + if len(flag_sets) > 1: + shared = set.intersection(*flag_sets.values()) + family_flags = {"--url", "--mode", "--category", "--pages", "--format", + "--out", "--delay", "--retries", "--retry-delay", + "--concurrency", "--proxy", "--proxy-file", + "--proxy-rotate", "--proxy-shuffle", + "--proxy-block-retries", "--twocaptcha-key", + "--captcha-api", "--solve-captcha", "--min-score", + "--cdp-endpoint", "--allow-empty", "--dump-html", + "--headless", "--headful"} + missing = family_flags - shared + ok &= check("every engine takes the family's flags (missing: %s)" + % (sorted(missing) or "none"), not missing) + + # A removed feature stays removed. Cheap, and it catches an editor + # reintroducing either. + for name, mod in loaded.items(): + source = inspect.getsource(mod) + for gone in ("--antidetect", "ANTIDETECT_LOCAL_API"): + ok &= check("%s does not reintroduce %s" % (name, gone), + gone not in source) + return ok + + +# --------------------------------------------------------------------------- +# Wording and shipped samples +# --------------------------------------------------------------------------- +# Enforced because the naming predates this repo and drifts back easily. The +# right name for the product is the Scraping Browser API; gate.2prx.com is +# not a host; the antidetect flag was removed because its endpoint was a +# placeholder. +BANNED_PHRASES = ("cloud browser", "antidetect browser", "Antidetect Browser", + "gate.2prx.com", "--antidetect", "ANTIDETECT_LOCAL_API", + "2parser") + +SHIPPED_TEXT_FILES = ("README.md", "TROUBLESHOOTING.md", "CONTRIBUTING.md", + "SECURITY.md", "CHANGELOG.md", ".env.example", + "CLAUDE.md") + + +def test_wording(): + group("wording") + ok = True + checked = 0 + for name in os.listdir(REPO_ROOT): + if not (name.endswith(".py") or name in SHIPPED_TEXT_FILES): + continue + if name == "smoke_test.py": + continue # this file names the banned phrases in order to ban them + path = os.path.join(REPO_ROOT, name) + if not os.path.isfile(path): + continue + text = open(path, encoding="utf-8", errors="replace").read() + checked += 1 + for phrase in BANNED_PHRASES: + if phrase in text: + ok &= check("%s must not contain %r" % (name, phrase), False) + ok &= check("banned-phrase scan ran over %d shipped files" % checked, checked > 5) + + # Positive check: the README must name the product correctly at least once. + readme = open(os.path.join(REPO_ROOT, "README.md"), encoding="utf-8").read() + ok &= check("README names the Scraping Browser API", + "Scraping Browser API" in readme) + ok &= check("README does not integrate a competitor solver", + not re.search(r"anti-?captcha|capmonster|deathbycaptcha", + readme, re.IGNORECASE)) + return ok + + +def test_sample_output(): + group("shipped sample output") + ok = True + sample_json = os.path.join(REPO_ROOT, "sample_output.json") + sample_csv = os.path.join(REPO_ROOT, "sample_output.csv") + if not (os.path.exists(sample_json) and os.path.exists(sample_csv)): + ok &= check("sample_output.json and sample_output.csv are shipped", False) + return ok + + rows = json.load(open(sample_json, encoding="utf-8")) + ok &= check("sample_output.json is a non-empty list of rows", + isinstance(rows, list) and rows) + expected = [f.name for f in fields(Product)] + ok &= check("sample_output.json columns match the Product schema exactly", + all(list(r.keys()) == expected for r in rows)) + header = open(sample_csv, encoding="utf-8").read().splitlines()[0] + ok &= check("sample_output.csv header matches the Product schema", + header.split(",") == expected) + # A sample cut from a real run, not typed by hand. + text = json.dumps(rows) + ok &= check("the sample carries no fabrication markers", + not re.search(r"example\.com|lorem ipsum|FIXME|TODO|XXXX", + text, re.IGNORECASE)) + ok &= check("every sample row has a real 10-character ASIN", + all(re.fullmatch(r"[A-Z0-9]{10}", r.get("sku") or "") for r in rows)) + ok &= check("the sample says which marketplace it came from", + all((r.get("source") or "").startswith("amazon.") for r in rows)) + return ok + + +# --------------------------------------------------------------------------- +def main() -> int: + ok = True + # Checks that could not run because an optional engine library is absent. + # Reported at the end: a suite that silently skips part of itself and + # still says "all passed" is the same defect as code that reports success + # without checking that what it wanted actually happened. + skips = [] + + ok &= test_price_parsing() + ok &= test_search_tiles() + ok &= test_localised_tiles() + ok &= test_bestseller_cards() + ok &= test_url_fallback_and_tile_scope() + ok &= test_product_detail() + ok &= test_reviews() + ok &= test_urls() + ok &= test_marketplaces() + ok &= test_page_state() + ok &= test_output_contract() + ok &= test_writers() + ok &= test_diff() + ok &= test_captcha() + ok &= test_page_flow() + ok &= test_env_config() + ok &= test_proxy_pool() + ok &= test_engines(skips) + ok &= test_wording() + ok &= test_sample_output() + + print() + if _failures: + print("%d check(s) FAILED:" % len(_failures)) + for f in _failures: + print(" - %s" % f) + if skips: + print("%d engine group(s) SKIPPED — an optional engine library is " + "absent. CI's engine-smoke job installs all three and fails if " + "this list is non-empty, because a skip reads exactly like a " + "passing run:" % len(skips)) + for s in skips: + print(" - %s" % s) + print("smoke_test: %s" % ("OK" if ok else "FAILED")) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..3124cd8 --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,27 @@ +""" +tests/test_smoke.py +-------------------- +A pytest entry point over the project's own offline suite. + +smoke_test.py (repo root) is deliberately a single self-contained runner with +inline HTML/JSON fixtures, not a pytest suite — see CONTRIBUTING.md for why. +This wraps it as one test rather than reimplementing its ~90 checks as pytest +asserts, so `pytest` and `python3 smoke_test.py` exercise the exact same code +path instead of two suites that can silently drift apart. Run +`python3 smoke_test.py` directly for per-check PASS/FAIL output; this test +only reports pass/fail as a whole, with that output attached on failure. +""" + +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def test_smoke_suite(): + result = subprocess.run( + [sys.executable, "smoke_test.py"], + cwd=REPO_ROOT, capture_output=True, text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr