diff --git a/.dockerignore b/.dockerignore index 053825e..cc37e39 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,13 @@ .git/ .github/ +# EVERY .env variant: a credential baked into an image is published to +# everyone who can pull it. .env +.env.* +*.bak # A downloaded proxy list is a credential file, one login and password # per line. The Dockerfile COPYs an explicit list rather than `COPY . .`, -# so this cannot get in today -- but a defence that depends on nobody ever +# so this cannot get in today — but a defence that depends on nobody ever # writing `COPY . .` is not a defence. Same narrow glob as .gitignore. proxylist*.txt *.log @@ -16,7 +20,13 @@ __pycache__/ .venv/ venv/ env/ -bershka_products.* -canary_run.* +# Scrape output, dumps and research captures. +live/ +recon/ +captures/ +canary.* +canary*.meta.json +*.latest_attempt.meta.json *_debug.html *_debug.png +*_p[0-9]*.html diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 2e3422a..2e4522a 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -2,14 +2,15 @@ """ CI checks that are too long to live inside the workflow YAML. -A workflow step CALLS these; it never re-implements them (family CLAUDE.md -§11). A heredoc in YAML is a check nobody runs locally and nobody notices +A workflow step CALLS these; it never re-implements them (a 2scraper family +rule). A heredoc in YAML is a check nobody runs locally and nobody notices going stale. Run any of these from the repo root: python .github/ci_checks.py --help-check python .github/ci_checks.py --sample-check python .github/ci_checks.py --secret-check python .github/ci_checks.py --run-check --output-prefix live/canary + python .github/ci_checks.py --wheel-check python .github/ci_checks.py --all Each prints what it looked at and exits non-zero on failure. @@ -174,8 +175,73 @@ def run_check(prefix="sample_output"): return failed +WHEEL_PROBE = r""" +import sys, product_parser as P, output_writer as O, page_flow +problems = [] +rules = P._robots_rules() +if len(rules) < 10: + problems.append('robots rules in the installed wheel: %d' % len(rules)) +for url, allowed in (('https://www.avito.ru/moskva?f=x', False), + ('https://www.avito.ru/moskva/telefony?s=104', False), + ('https://www.avito.ru/moskva/telefony?p=2', True)): + if P.robots_verdict(url)[0] is not allowed: + problems.append('robots verdict for %s is not %s' % (url, allowed)) +if O.SOURCE_DEFAULT != 'avito.ru': + problems.append('source is %r' % O.SOURCE_DEFAULT) +if P.page_url('https://www.avito.ru/moskva/telefony', 3) != 'https://www.avito.ru/moskva/telefony?p=3': + problems.append('pagination convention did not survive packaging') +if not page_flow.should_solve('captcha'): + problems.append('state policy did not survive packaging') +print('\n'.join(problems)) +sys.exit(1 if problems else 0) +""" + + +def wheel_check(with_playwright=True): + """Build the wheel, install it OUTSIDE the checkout, and use it as a user + would. v1.0.0 passed an import-only version of this while its wheel + shipped no robots rules (every URL allowed) and its console command died + on a missing playwright (external audit, 2026-10-07). + """ + import os + import tempfile + failed = [] + with tempfile.TemporaryDirectory() as tmp: + dist = os.path.join(tmp, "dist") + venv = os.path.join(tmp, "venv") + step = subprocess.run([sys.executable, "-m", "pip", "wheel", "--no-deps", "-q", + "-w", dist, str(REPO)], capture_output=True, text=True) + if step.returncode: + return ["wheel build failed: " + step.stderr[-400:]] + subprocess.run([sys.executable, "-m", "venv", venv], check=True) + py = os.path.join(venv, "bin", "python") + cli = os.path.join(venv, "bin", "avito-scraper") + wheel = os.path.join(dist, os.listdir(dist)[0]) + subprocess.run([py, "-m", "pip", "install", "-q", wheel], check=True) + probe = subprocess.run([py, "-c", WHEEL_PROBE], capture_output=True, text=True, cwd=tmp) + if probe.returncode: + failed += ["installed wheel: " + l for l in probe.stdout.splitlines() + [probe.stderr[-300:]] if l] + else: + print("ok installed wheel: robots rules, verdicts, conventions") + bare = subprocess.run([cli, "--help"], capture_output=True, text=True, cwd=tmp) + if bare.returncode != 2 or "avito-scraper[playwright]" not in bare.stderr: + failed.append("console script without playwright: rc=%s, %r" + % (bare.returncode, (bare.stderr or bare.stdout)[-200:])) + else: + print("ok avito-scraper without playwright: says what to install (exit 2)") + if with_playwright: + subprocess.run([py, "-m", "pip", "install", "-q", wheel + "[playwright]"], check=True) + full = subprocess.run([cli, "--help"], capture_output=True, text=True, cwd=tmp) + if full.returncode != 0 or "--mode" not in full.stdout: + failed.append("console script with playwright: rc=%s, %r" + % (full.returncode, full.stderr[-200:])) + else: + print("ok avito-scraper[playwright]: --help runs") + return failed + + CHECKS = {"help": help_check, "sample": sample_check, "secret": secret_check, - "run": run_check} + "run": run_check, "wheel": wheel_check} def main(): @@ -193,12 +259,14 @@ def main(): "(use with --output-prefix)") parser.add_argument("--secret-check", action="store_true", help="No credentials committed anywhere") + parser.add_argument("--wheel-check", action="store_true", + help="Build the wheel, install it outside the checkout, use it") parser.add_argument("--all", action="store_true", - help="help + sample + secret (the run check needs a live run)") + help="help + sample + secret (run needs a live run; wheel is slow)") args = parser.parse_args() selected = [name for name in CHECKS - if (args.all and name != "run") or getattr(args, f"{name}_check")] + if (args.all and name not in ("run", "wheel")) or getattr(args, f"{name}_check")] if not selected: parser.error("pick at least one check, or --all") failures = [] diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index c96994f..ce73334 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -43,7 +43,8 @@ jobs: - name: Explain the skip if: env.READY != 'true' run: | - echo "::notice::Skipped: needs secrets TWOCAPTCHA_KEY and AVITO_PROXY and the" + echo "::warning::LIVE RUN SKIPPED — this green run proves the workflow is healthy, NOT that avito.ru was scraped." + echo "::notice::Needs secrets TWOCAPTCHA_KEY and AVITO_PROXY and the" echo "::notice::variable AVITO_BROWSER_ACCOUNT_ID. A GitHub runner is not a" echo "::notice::Russian residential address; a live run from here without the" echo "::notice::Scraping Browser would be red every day regardless of the code." @@ -87,7 +88,7 @@ jobs: - name: Assert the data, not just the exit code if: env.READY == 'true' - # THE SHIPPED CHECK, not an inline copy (CLAUDE.md §11). + # THE SHIPPED CHECK, not an inline copy (a 2scraper family rule). run: | echo "exit code from the run: $rc" python3 .github/ci_checks.py --run-check --sample-check --output-prefix canary diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 9c0337f..8788b74 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -77,17 +77,20 @@ jobs: you can find is a number or a claim that no longer matches the code, or a comment that explains a reason the code no longer has. - A few things specific to this repo worth checking: every price is - in EUR and every row is a DOM reading (there is no JSON-LD on - this site to cross-check against — see output_writer.py and - diff_runs.py); there are two row schemas, Player and Transfer - (output_writer.py), and a change to one should not silently affect - the other; `page` and `row_index` must stay unique across a - paginated run (see product_parser.py and the tokopedia lesson in - CLAUDE.md); and this site's listing pagination is robots-DISALLOWED, - so product_parser.page_url() returning its input unchanged is a - deliberate decision, not a stub — a change that makes it - construct a URL deserves extra scrutiny. + A few things specific to this repo (avito.ru) worth checking: + `price` must stay the DISPLAYED price — Avito's bare price fields + are the pre-discount figure (product_parser.py docstring); a + listing page arrives in two renderings, DOM and an embedded JSON + state, and `price_source` says which, so diff_runs.py must not + report a rendering change as a price change; there are three row + schemas, Product / Item / Seller (output_writer.py), and Item must + keep Product's column prefix; `page` and `position` must stay + unique across a paginated run; a captcha solve must go through the + browser's own proxy (captcha_solver.py — a token solved from + another IP is rejected); a transport fault is exit 5, never 3; + phone numbers, IMEI and a private seller's placeholder name must + never reach a row; and a URL robots_snapshot.py disallows must be + refused before any request. Also worth flagging: a check that cannot fail, an error path that swallows a failure silently, a credential that could reach a diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2ca471c..328235b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,7 +43,7 @@ jobs: pytest -q - name: Every CLI answers --help - # THE SHIPPED CHECK, not a copy of it (CLAUDE.md §11). Engine + # THE SHIPPED CHECK, not a copy of it (a 2scraper family rule). Engine # libraries are not installed here, so those entry points report # skipped rather than failed. run: python3 .github/ci_checks.py --help-check @@ -55,29 +55,13 @@ jobs: run: python env_config.py - name: The built wheel installs and works outside the checkout - # A flat `py-modules` collection is easy to get wrong, and a wheel - # that imports in the source tree but not once installed fails only - # on delivery. Installed into its own venv, exercised from /tmp. - run: | - python -m pip install --quiet build - python -m build --wheel - python -m venv /tmp/wheelcheck - /tmp/wheelcheck/bin/pip install --quiet dist/*.whl - cd /tmp - /tmp/wheelcheck/bin/python -c " - import sys, output_writer as O, page_flow, product_parser as P, captcha_solver, proxy_forwarder - problems = [] - if O.SOURCE_DEFAULT != 'avito.ru': - problems.append('source is %r' % O.SOURCE_DEFAULT) - u = 'https://www.avito.ru/moskva/telefony' - if P.page_url(u, 3) != u + '?p=3': - problems.append('pagination convention did not survive packaging') - if not page_flow.should_solve('captcha'): - problems.append('state policy did not survive packaging') - for p in problems: - print('::error::' + p) - print('installed wheel: %d problem(s)' % len(problems)) - sys.exit(1 if problems else 0)" + # THE SHIPPED CHECK. Builds the wheel, installs it into a fresh venv + # outside the checkout and uses it: robots rules present and + # refusing, the console command without playwright says what to + # install, and with it answers --help. v1.0.0 passed an import-only + # inline version of this while its wheel allowed every URL and its + # command died on a missing playwright (external audit, 2026-10-07). + run: python3 .github/ci_checks.py --wheel-check - name: No credentials committed # THE SHIPPED CHECK — one scanner, shared with the git hooks (§21). @@ -90,7 +74,7 @@ jobs: # # ONE VENV PER ENGINE: the engines' pins are mutually unsatisfiable # (pyee, urllib3), and installed together pip may resolve the conflict by - # picking a release nobody uses (CLAUDE.md §11). + # picking a release nobody uses (a 2scraper family rule). runs-on: ubuntu-latest strategy: fail-fast: false @@ -168,7 +152,8 @@ jobs: import importlib, sys mods = ['browser_bridge','captcha_solver','diff_runs','env_config', 'fingerprint_client','output_writer','page_flow', - 'playwright_scraper','product_parser','proxy_forwarder','proxy_pool'] + 'playwright_scraper','product_parser','proxy_forwarder','proxy_pool', + 'robots_snapshot'] bad = [] for m in mods: try: diff --git a/.gitignore b/.gitignore index a11d5fb..305f40e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,59 +1,49 @@ -# Everything below is GENERATED by running this project. None of it belongs in -# git — it is either large, machine-specific, or a snapshot of a site that will -# have changed by the time anyone reads it. A single 3-page run drops several -# MB of HTML snapshots alone. +# Everything below is GENERATED by running this project, or is a credential. +# None of it belongs in git. # --- scrape output --------------------------------------------------------- *.json *.csv -!package.json -# Committed on purpose: a real excerpt of a real run (--mode market-values), -# so a reader can see the exact output shape without installing anything. +# Committed on purpose: a real, scrubbed excerpt of a real listing run, so a +# reader can see the output shape without installing anything. Its metadata +# sidecar travels with it. !sample_output.json -# The sample's run-metadata sidecar travels with it. `.github/ci_checks.py -# --run-check` reads it, so a fresh checkout has to carry it or `--all` -# fails on a file the repo deliberately withheld. -!sample_output.meta.json !sample_output.csv -homedepot_products.* -bershka_products_*.* -# The canary workflow's own run, and anyone reproducing it locally. -canary_run.* +!sample_output.meta.json +*.latest_attempt.meta.json +avito.* +avito_listings.* +# This repo's own working output, and the canary's. +live/ +canary.json +canary.csv +canary*.meta.json -# --- debug artifacts the engines drop next to themselves ------------------- -# Written on purpose when a page yields nothing or comes back blocked, and by -# --dump-html on success too. Large, and stale within hours. +# --- debug artifacts --------------------------------------------------------- +# --dump-html writes the page on success too. A dump carries the session that +# fetched it and sellers' personal details. *_debug.html *_debug.png *.html.page* +*_p[0-9]*.html -# --- unlike other members of this family, index.html and landing.md are ---- -# --- committed here on purpose, not generated -- do NOT gitignore them ----- -# (kept as an explicit note rather than a silent omission, since every sibling -# scraper in this family DOES gitignore an index.html as "published -# elsewhere", and that is deliberately not true of this repo) +# --- raw research captures --------------------------------------------------- +recon/ +captures/ +artifacts/ -# --- a downloaded 2captcha proxy list is a credential file ------------------ -# Every line of one carries a login and a password. The glob covers the -# variants people actually make (proxylist-us.txt, proxylist.2.txt) and -# stops there: a broader shape such as *proxy*.txt would also match a file -# someone meant to commit, and .gitignore has no undo. Every proxy filename -# this project's own documentation shows must match this -- checked in the -# suite with `git check-ignore`, not by comparing strings, because -# .gitignore has patterns, negations and directory scoping and a substring -# match proves nothing. +# --- a downloaded 2captcha proxy list is a credential file ------------------- +# Every line carries a login and a password. The glob covers the variants +# people make (proxylist-ru.txt, proxylist.2.txt) and stops there. proxylist*.txt # --- caches ------------------------------------------------------------------ __pycache__/ *.py[cod] .pytest_cache/ -# fingerprint_client.py's own on-disk cache defaults to -# ~/.cache/2captcha-fingerprints (outside the repo) -- this only matters if -# --cache-dir ever points somewhere inside it. .fingerprint_cache/ -# --- packaging build artifacts ---------------------------------------------- +# --- packaging build artifacts ----------------------------------------------- *.egg-info/ dist/ build/ @@ -63,51 +53,21 @@ build/ venv/ env/ -# --- editors / OS -------------------------------------------------------- +# --- editors / OS ------------------------------------------------------------ .DS_Store *.swp .idea/ .vscode/ +.claude/ -# --- never, under any circumstances -------------------------------------- -# The scrapers mask user:pass in their own logs, but raw HTML dumps are NOT -# masked, and a key pasted into a shell history file is not masked either. +# --- never, under any circumstances -------------------------------------------- +# EVERY .env variant, not just `.env`: a `.env.user-endpoint.bak` was committed +# in a sibling repo because `.gitignore` listed the exact name and the backup +# had a suffix. Only the documented example is exempt. *.log -.env *.key secrets* - -# --- local research material ------------------------------------------------ -# `captures/` holds the raw pages this repo was built against: 28MB of real -# HTML and sitemaps from bershkabeauty.com, fetched 2026-09-17/18. It is -# EVIDENCE, kept on disk so a claim in the README can be re-checked, but it -# is not repo content -- the fixtures the test suite actually runs against -# are trimmed, scrubbed excerpts embedded in smoke_test.py, per CLAUDE.md -# §10. A committed capture would also carry whatever per-visitor state the -# page happened to hold. -captures/ - -# This repo's own working output -live/ -canary.json -canary.csv -canary*.meta.json -homedepot_products.* -*.latest_attempt.meta.json -!sample_output.meta.json - -# Raw captures. A page dump carries the session that fetched it. -recon/ -*_debug.html -*_debug.png -artifacts/ - -# EVERY .env variant, not just `.env`. A `.env.user-endpoint.bak` was committed -# on 2026-09-22 because `.gitignore` listed the exact name and the backup had a -# suffix. Match the family, exempt only the documented example. .env .env.* !.env.example *.bak - -.claude/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 27274c8..c3f8283 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,55 @@ follow SemVer as closely as a CLI toolkit can: a **patch** means fixes, and a fix may change a default when the old one was wrong — such a change leads its entry in a blockquote. +## [1.0.1] — 2026-10-07 + +Fixes from an external audit of v1.0.0. Each finding was reproduced before +it was fixed, and each now has a regression test. + +> **Behaviour changes for existing users.** +> * An **unexpected exception** now exits **1** (crash, traceback printed +> with credentials masked). It used to exit 5, the same as a dead exit, so a +> parser bug read as a transport fault. Exit 5 is now only a transport fault. +> * `diff_runs.py` now **refuses** two outputs when either has no readable +> `.meta.json`, or when they were written by different row schemas. Pass +> `--force` to compare anyway. + +### Fixed + +* **The installed wheel allowed every URL.** The robots snapshot was a loose + `.txt` beside the modules and was not shipped; an installed copy read zero + rules. It is now the module `robots_snapshot.py`, and an empty rule set + raises instead of allowing everything. +* **`pip install avito-scraper` gave a command that crashed** with + `ModuleNotFoundError: playwright`. The command now goes through a launcher + that says to install `avito-scraper[playwright]` (exit 2). +* **No attempt metadata on a failed start or a crash.** + `.latest_attempt.meta.json` is now written on every ending, with + `exit_code`, `error_class` and a masked `error_message`. +* **`schema_version`** now fingerprints the row class of the run's mode + (names, order and types), not `Product`'s field names only. +* Leftovers from the repositories this one was scaffolded from, in + `requirements.txt`, `.gitignore`, `.dockerignore` and the Claude review + prompt, are gone; the vocabulary check now covers every tracked file. + Comments no longer point at an operator file that is not published. + +### Changed + +* CI builds the wheel, installs it outside the checkout and uses it + (`ci_checks.py --wheel-check`): robots verdicts, and the console command + with and without the playwright extra. +* A skipped canary (no secrets — the case for this public repository) now + raises a visible warning; README explains what the green badge does and + does not prove. + +### Not changed, and why + +* **The canary still goes green when it skips.** A check that is red every + day for want of secrets teaches everyone to ignore checks; the skip is now + loud instead. +* Pinning Actions by SHA, lock files, a `src/` package layout and dropping + pyppeteer are larger decisions, left for a minor release. + ## [1.0.0] — 2026-10-06 First release. diff --git a/Dockerfile b/Dockerfile index 9f6d792..2a9f843 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx # every invocation, --help included, because nothing ever built it. COPY browser_bridge.py captcha_solver.py diff_runs.py env_config.py \ fingerprint_client.py output_writer.py page_flow.py playwright_scraper.py \ - product_parser.py proxy_forwarder.py proxy_pool.py robots.snapshot.txt ./ + product_parser.py proxy_forwarder.py proxy_pool.py robots_snapshot.py ./ ENTRYPOINT ["python3", "playwright_scraper.py"] CMD ["--help"] diff --git a/README.md b/README.md index 4d69163..fb58e33 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,13 @@ actually needs. [![engines](https://img.shields.io/badge/engines-Playwright%20%7C%20Selenium%20%7C%20pyppeteer%20%7C%20CDP-informational)](#engines) [![needs a Russian exit](https://img.shields.io/badge/needs-a%20Russian%20exit%20%2B%20a%20browser-orange)](#the-one-thing-you-actually-need) +> **What the canary badge means.** The daily canary scrapes avito.ru only when +> the repository holds the secrets for a Russian exit and a captcha key. This +> public repository does not, so the canary takes its documented SKIP branch: +> a green badge here says the workflow is healthy, **not** that avito.ru was +> scraped today. Each skipped run says so in a warning. The live measurements +> are in [Measured results](#measured-results), with their dates. + --- ## The one thing you actually need @@ -248,7 +255,7 @@ pyppeteer is effectively unmaintained; it is here for parity. | Code | Meaning | |---|---| | `0` | Complete | -| `1` | Crash | +| `1` | Crash — an unexpected exception: a bug, not the site and not the transport (traceback printed, credentials masked) | | `2` | Bad usage — wrong host, wrong `--mode` for the URL, a robots-disallowed URL | | `3` | Blocked — the firewall stayed up after every rung | | `4` | Zero results — including a search with no matches | @@ -260,7 +267,7 @@ the answer is "run it again" or "rebuild the endpoint", not the firewall. A run that finds nothing writes nothing — last night's output is not replaced with `[]` (`--allow-empty` is the opt-out). `.meta.json` describes the -data beside it; `.latest_attempt.meta.json` is written on every run and +data beside it; `.latest_attempt.meta.json` is written on EVERY ending — success, block, transport error or crash, with `exit_code` and `error_class` — and records `transport_facts`: walls met, which rung cleared each, every solve with its time and cost, rotations, and auto-solve events. @@ -349,7 +356,7 @@ details, and does not message sellers. When avito.ru's firewall presents a captcha, the scraper solves it the way a visitor would — through the page's own form — with a solve you pay 2Captcha for. -`robots.txt` is respected: the snapshot is in `robots.snapshot.txt`, and URLs +`robots.txt` is respected: the snapshot is in `robots_snapshot.py`, and URLs it disallows are refused before any request is made. The firewall exists because of request volume. Default `--delay` is 2 seconds diff --git a/avito_cli.py b/avito_cli.py new file mode 100644 index 0000000..3d6dfb4 --- /dev/null +++ b/avito_cli.py @@ -0,0 +1,30 @@ +"""The `avito-scraper` console command. + +A thin launcher on purpose. The command runs the Playwright engine, but +Playwright is an optional extra (`avito-scraper[playwright]`): the three +engines' pins conflict, so none of them can be a base dependency. Importing +the engine at module level here made a plain `pip install avito-scraper` +produce a command that died with `ModuleNotFoundError: playwright` on its +first invocation, `--help` included (found by an external audit, +2026-10-07). This says what to install instead. +""" + +import sys + +INSTALL_HINT = ("avito-scraper runs the Playwright engine, which is an optional extra:\n" + " pip install 'avito-scraper[playwright]' && playwright install chromium") + + +def main(argv=None) -> int: + try: + import playwright_scraper + except ModuleNotFoundError as exc: + if (exc.name or "").split(".")[0] == "playwright": + print(INSTALL_HINT, file=sys.stderr) + return 2 + raise + return playwright_scraper.main(argv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/browser_bridge.py b/browser_bridge.py index 6882220..3766aca 100644 --- a/browser_bridge.py +++ b/browser_bridge.py @@ -71,7 +71,7 @@ import page_flow import product_parser as P import proxy_pool -from output_writer import (EXIT_REMOTE_API_ERROR, Product, finish_run, +from output_writer import (EXIT_CRASH, EXIT_REMOTE_API_ERROR, Product, finish_run, scope_fingerprint) logger = logging.getLogger("avito") @@ -93,7 +93,7 @@ class BridgeError(RuntimeError): exit (measured once on a local run), a locked profile, a 407 on the tunnel: transport faults, not refusals. They map to exit 5, never 3 — exit 3 sends the reader to the anti-bot problem when the answer is "run - it again" (CLAUDE.md §20). + it again" (2scraper family rule). """ @@ -186,7 +186,7 @@ def solve_wall(driver, args, url: str, html: str, facts: Dict[str, Any], log=Non """Rungs 1 and 2 on a captcha wall. Returns fresh HTML if it cleared. Every solver failure is a WARNING: the run goes on and reports exit 3 if - the page stays walled (CLAUDE.md §8). + the page stays walled (2scraper family rule). """ if getattr(args, "solve_captcha", "when-blocked") == "never": return None @@ -255,7 +255,7 @@ def fetch_page(driver, args, url: str, mode: str, facts: Dict[str, Any], log=Non status = driver.navigate(url) break except Exception as exc: # noqa: BLE001 - # A transport fault: same exit, another try (CLAUDE.md §8 — a + # A transport fault: same exit, another try (2scraper family rule — a # timeout and a refusal want opposite responses). message = mask_text(str(exc)).splitlines()[0][:200] if str(exc) else "" if attempt == attempts: @@ -297,7 +297,7 @@ def fetch_page(driver, args, url: str, mode: str, facts: Dict[str, Any], log=Non def fetch_with_rotation(driver, args, url: str, mode: str, facts, log=None) -> Dict[str, Any]: """`fetch_page`, plus rung 3: a fresh exit for a page that stayed walled. - A rotation is a FRESH BROWSER (CLAUDE.md §8): cookies a firewall issued to + A rotation is a FRESH BROWSER (2scraper family rule): cookies a firewall issued to exit A replayed from exit B are a stronger signal than either alone. """ retries = getattr(args, "proxy_block_retries", 0) or 0 @@ -306,7 +306,7 @@ def fetch_with_rotation(driver, args, url: str, mode: str, facts, log=None) -> D result = fetch_page(driver, args, url, mode, facts, log=log) except BridgeError as exc: # A DEAD EXIT wants a different exit, not another try at the same - # one (CLAUDE.md §8). Measured 2026-10-06: a session-pinned + # one (2scraper family rule). Measured 2026-10-06: a session-pinned # residential exit went away inside its sessTime and every # request through it failed with ERR_CONNECTION_CLOSED / # SSL_ERROR_SYSCALL, while a fresh session on the same credential @@ -444,12 +444,40 @@ def run(args, driver) -> int: seen: set = set() start_url = urls[0] + pages_requested = args.pages if args.mode == "listing" else len(urls) + + def failed(code: int, exc: BaseException, stop_reason: str) -> int: + """EVERY ending writes the attempt sidecar — a run that died before + finish_run used to leave the previous attempt's metadata in place, so + a scheduler read yesterday's `complete` as today's result.""" + facts["error_class"] = type(exc).__name__ + facts["error_message"] = mask_text(str(exc))[:500] + meta = output_writer.run_meta( + status="failed", stop_reason=stop_reason, pages_requested=pages_requested, + pages_completed=facts["pages_completed"], start_url=start_url, + final_url=facts.get("final_url", start_url), products=0, + pages_failed=facts["failed_pages"], mode=args.mode, + scope=scope_fingerprint(start_url, args.mode, pages_requested, args.max_products)) + meta.update(run_id=run_id, started_at=started_at, blocked=False, + exit_code=code, data_updated=False, transport_facts=facts) + try: + output_writer.write_attempt_meta(args.out, meta) + except OSError as write_exc: + print("[!] could not write the attempt metadata: %s" % write_exc, file=sys.stderr) + return code + + import uuid + from datetime import datetime, timezone + run_id = uuid.uuid4().hex[:12] + started_at = datetime.now(timezone.utc).isoformat() + try: driver.start() - except Exception as exc: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 — a browser that never started is transport print("could not start %s: %s: %s" % (driver.name, type(exc).__name__, mask_text(str(exc))[:300]), file=sys.stderr) - return EXIT_REMOTE_API_ERROR + _stop(driver) + return failed(EXIT_REMOTE_API_ERROR, exc, "transport_error") try: if args.mode == "listing": @@ -459,14 +487,18 @@ def run(args, driver) -> int: else: rows, extra_rows = _run_seller(driver, args, start_url, facts, log) except BridgeError as exc: + # The site was never reached: exit 5 (2scraper family rule). print("[!] %s" % exc, file=sys.stderr) - facts["transport_error"] = str(exc) _stop(driver) - return EXIT_REMOTE_API_ERROR + return failed(EXIT_REMOTE_API_ERROR, exc, "transport_error") except Exception as exc: # noqa: BLE001 - print("[!] %s: %s" % (type(exc).__name__, mask_text(str(exc))[:300]), file=sys.stderr) + # NOT a transport fault: a parser bug, a TypeError, a file error. It + # used to be reported as exit 5, indistinguishable from a dead exit, + # with the traceback thrown away. Exit 1, traceback kept (masked). + import traceback + print(mask_text(traceback.format_exc()), file=sys.stderr) _stop(driver) - return EXIT_REMOTE_API_ERROR + return failed(EXIT_CRASH, exc, "crash") finally: _stop(driver) @@ -479,7 +511,6 @@ def run(args, driver) -> int: output_writer.save(extra_rows, args.out + "_listings", args.format, allow_empty=False, row_cls=Product) - pages_requested = args.pages if args.mode == "listing" else len(urls) return finish_run( rows, args.out, args.format, args.allow_empty, blocked=facts["blocked"], @@ -540,7 +571,7 @@ def _run_listing(driver, args, url, facts, seen, log) -> List[Product]: facts["pages_completed"] = page log(" %s: %d rows (%d new, %d total)" % (state, len(page_rows), len(fresh), len(rows))) - # The terminating condition is DATA, never a selector (CLAUDE.md §7). + # The terminating condition is DATA, never a selector (2scraper family rule). if not page_rows or not fresh: log(" page %d added no new items — end of listing." % page) facts["pagination_exhausted"] = True diff --git a/captcha_solver.py b/captcha_solver.py index 1380c74..d85a2de 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -21,7 +21,7 @@ token (see below) These are statements about THIS repo, not about what 2Captcha can solve -(CLAUDE.md §19). +(2scraper family rule). The token is bound to the exit — measured ----------------------------------------- @@ -35,7 +35,7 @@ So the solve MUST go through the same exit the browser uses. `solve_geetest_v4` refuses to build a proxyless task unless asked to, because paying for a token the site rejects — and then facing a second captcha — is worse than -reporting the page unsolved (CLAUDE.md §8: detected != paying). +reporting the page unsolved (2scraper family rule: detected != paying). Cost measured: $0.00299 per solve, 10–73 s. @@ -177,7 +177,7 @@ def firewall_response_value(solution: dict) -> str: # Submitting through the PAGE'S OWN form rather than calling the verify # endpoint ourselves: the form handler adds the `X-Cube` header from # `#cubeResult` and does the reload, exactly as it does for a person. -# Each engine wraps this body in its own dialect (CLAUDE.md §1). +# Each engine wraps this body in its own dialect (2scraper family rule). SUBMIT_JS_BODY = ( "var inp = document.querySelector('input[name=captcha-response]');" "var form = document.querySelector('.js-firewall-form');" diff --git a/diff_runs.py b/diff_runs.py index 37f8c71..5d6b838 100644 --- a/diff_runs.py +++ b/diff_runs.py @@ -32,7 +32,7 @@ renderings would otherwise report every discounted item as "original_price 69989 -> None" — our instrument changing, not the item. So when the two rows were read differently, the fields only one rendering states are not -compared, and a price difference goes in its own bucket (CLAUDE.md §8). +compared, and a price difference goes in its own bucket (2scraper family rule). A run is also refused against a run of a DIFFERENT listing (another city or category, from the sidecar's `scope.listing`): every row would read as added @@ -168,11 +168,19 @@ def _check_comparable(args) -> bool: problems = [] modes = {} listings = {} + schemas = {} for label, path in (("--old", args.old), ("--new", args.new)): status, meta = _run_status(path) if status is None: + # No readable sidecar means no proof the run was complete, or of + # its mode and scope — exactly what this check exists to prove. + # It used to be skipped, so two bare JSON files compared freely. + problems.append( + f"{label} ({path}) has no readable .meta.json beside it, so " + f"nothing shows it was complete or what it covered.") continue scope = (meta or {}).get("scope") or {} + schemas[label] = scope.get("schema_version") if scope.get("listing"): # The page count is part of what a listing run COVERED: page 3 # of one run is simply absent from a 2-page run. @@ -198,6 +206,10 @@ def _check_comparable(args) -> bool: problems.append( f"the two runs covered different listings or page counts ({listings}): " f"rows outside the overlap would read as added or removed.") + if len(set(schemas.values())) > 1: + problems.append( + f"the two runs were written by different row schemas ({schemas}): " + f"a renamed or retyped column would read as a change on every row.") if not problems: return True diff --git a/env_config.py b/env_config.py index 1428dfb..3e4d7ee 100644 --- a/env_config.py +++ b/env_config.py @@ -69,7 +69,7 @@ # cause. This exact defect shipped in this family's etsy-scraper and # a sibling repo (both of which use braced placeholders) and was absent # from amazon-scraper/a sibling repo only because neither of those uses -# them -- see CLAUDE.md §17. Any `{...}` left in a value is unset here, +# them -- see 2scraper family rule. Any `{...}` left in a value is unset here, # named rather than quoted in the warning, since the value can be a # credentialled URL. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[^{}]+\}") diff --git a/fingerprint_client.py b/fingerprint_client.py index c25a6b3..2d954dc 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -335,7 +335,7 @@ def main() -> int: # same CLI > env > .env > default precedence every other entry point in # this repo uses -- this file used to be the one CLI that skipped it, # which meant `python3 fingerprint_client.py` (the first thing to run - # when a key "isn't working", per CLAUDE.md) could not see a .env-only + # when a key "isn't working", per this family's convention) could not see a .env-only # key at all. p.add_argument("--key", default=None, help="API key. Falls back to $TWOCAPTCHA_KEY via .env/" diff --git a/output_writer.py b/output_writer.py index afa3a8a..d0d41a7 100644 --- a/output_writer.py +++ b/output_writer.py @@ -13,7 +13,7 @@ The first nine `Product` columns are the family prefix — `source`, `scraped_at`, `url`, `sku`, `title`, `image_url`, `price`, `currency`, `category` — byte-identical to every sibling repo, so a consumer written -against another repo in this family reads them unchanged (CLAUDE.md §9). +against another repo in this family reads them unchanged (2scraper family rule). `Item` extends `Product`, so its prefix is the same by construction. `sku` is Avito's item id (`8245354778`): the trailing digits of the item @@ -238,6 +238,9 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: EXIT_NO_PRODUCTS = 4 +# An unexpected exception — a bug, not the site and not the transport. +EXIT_CRASH = 1 + # Exit code for a run blocked by a bot-check/challenge page before parsing # even started. EXIT_BLOCKED = 3 @@ -249,7 +252,7 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = Product) -> None: # -- the Fingerprint API rejecting a request (bad key, bad --tags, rate # limit), or a Scraping Browser CDP connection failing (e.g. profile_locked) # -- as opposed to EXIT_BLOCKED (the TARGET SITE refusing a page) or an -# uncaught crash (1). Per CLAUDE.md's family exit-code contract ("5" = +# uncaught crash (1). Per the 2scraper family exit-code contract ("5" = # "remote API error"). Deliberately NOT what a captcha-solve failure gets: # per that same document's captcha section, a solver error is a WARNING that # lets the run continue (see captcha_solver.py and each engine's @@ -311,11 +314,17 @@ def write_attempt_meta(out_prefix: str, meta: dict) -> str: return write_run_meta(out_prefix, meta, suffix=ATTEMPT_META_SUFFIX) -def schema_version() -> str: - """A short fingerprint of the row schema, so a diff can refuse to compare - a run written before a column changed with one written after.""" - names = ",".join(f.name for f in fields(Product)) - return hashlib.sha256(names.encode()).hexdigest()[:12] +def schema_version(row_cls: Type = Product) -> str: + """A short fingerprint of ONE row schema — names, order and declared + types — so a diff can refuse to compare a run written before a column + changed with one written after. + + Per row class: the first version hashed `Product`'s field names only, so + an `Item`-only column change, or a type change, left it unchanged + (external audit, 2026-10-07). + """ + spec = ",".join("%s:%s" % (f.name, f.type) for f in fields(row_cls)) + return hashlib.sha256((row_cls.__name__ + "|" + spec).encode()).hexdigest()[:12] def scope_fingerprint(start_url: Optional[str] = None, mode: Optional[str] = None, @@ -338,7 +347,7 @@ def scope_fingerprint(start_url: Optional[str] = None, mode: Optional[str] = Non "mode": mode, "pages": pages, "max_products": max_products, - "schema_version": schema_version(), + "schema_version": schema_version(ROW_CLASS_BY_MODE.get(mode, Product)), } @@ -369,7 +378,7 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, """Write JSON/CSV and return a process exit code. On zero rows, nothing is written at all unless `allow_empty` — see the - family invariant in CLAUDE.md §8: a run that finds nothing must not + 2scraper family rule: a run that finds nothing must not silently replace yesterday's good output with an empty file. """ if not rows and not allow_empty: diff --git a/page_flow.py b/page_flow.py index ed63f7f..c3a942f 100644 --- a/page_flow.py +++ b/page_flow.py @@ -3,7 +3,7 @@ ------------ avito.ru's page-state policy, shared by all three engines as DATA, so an engine cannot quietly disagree with its twins about whether a page is worth -waiting for, paying for or rotating away from (CLAUDE.md §1). +waiting for, paying for or rotating away from (2scraper family rule). What a page can answer (product_parser.detect_page_state) -------------------------------------------------------- @@ -56,7 +56,7 @@ # Readiness anchors per mode — what "the page has painted" means. # A listing is ready when MORE than one card is present: one match can # resolve on an unrelated card-shaped node long before the grid paints -# (CLAUDE.md §5). An item page is ready when its hydration payload is in. +# (2scraper family rule). An item page is ready when its hydration payload is in. # A seller page is ready when the profile block is. READY_SELECTORS = { "listing": '[data-marker="item"][data-item-id]', @@ -80,7 +80,7 @@ def should_retry(state: str) -> bool: def should_rotate(state: str) -> bool: """Whether this state means "try a different address", not "try again" - (CLAUDE.md §8: a timeout and a refusal want opposite responses).""" + (2scraper family rule: a timeout and a refusal want opposite responses).""" return STATE_POLICY.get(state, {}).get("rotate", False) diff --git a/playwright_scraper.py b/playwright_scraper.py index 4836e3f..938e8f3 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -23,7 +23,7 @@ # Imported at MODULE level on purpose: the offline suite must be able to skip # this engine when Playwright is absent, and it can only tell that if the -# import fails here rather than deep inside start() (CLAUDE.md §10). +# import fails here rather than deep inside start() (2scraper family rule). from playwright.sync_api import sync_playwright @@ -47,7 +47,7 @@ def start(self): if over_cdp: # Wrapped so a failure cannot print the endpoint, which carries a # password. Playwright repeats it five times in one error — the - # message plus a four-line call log (CLAUDE.md §8). + # message plus a four-line call log (2scraper family rule). try: self._browser = self._pw.chromium.connect_over_cdp( self.args.cdp_endpoint, timeout=self.args.timeout * 1000) @@ -69,7 +69,7 @@ def start(self): self._browser = self._pw.chromium.launch(**launch) # No user agent and no fingerprint: a hardcoded UA drifts from the # installed Chromium, and over CDP the remote browser brings its - # own (CLAUDE.md §8). + # own (2scraper family rule). self._context = self._browser.new_context(locale="ru-RU") self._page = self._context.new_page() self._page.set_default_timeout(self.args.timeout * 1000) diff --git a/product_parser.py b/product_parser.py index c9add00..16756b3 100644 --- a/product_parser.py +++ b/product_parser.py @@ -1,7 +1,7 @@ """ product_parser.py ----------------- -Everything this repo knows about avito.ru's markup lives here (CLAUDE.md §1). +Everything this repo knows about avito.ru's markup lives here (2scraper family rule). The engines and `browser_bridge.py` decide WHEN to fetch; this module decides WHAT a fetched page says. @@ -171,25 +171,25 @@ def _robots_rules() -> List[Tuple[str, str]]: global _ROBOTS if _ROBOTS is not None: return _ROBOTS - import os - path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "robots.snapshot.txt") + from robots_snapshot import ROBOTS_TXT rules: List[Tuple[str, str]] = [] in_star = False - try: - with open(path, encoding="utf-8") as handle: - for raw in handle: - line = raw.split("#", 1)[0].strip() - if not line or ":" not in line: - continue - key, value = (s.strip() for s in line.split(":", 1)) - key = key.lower() - if key == "user-agent": - in_star = value == "*" - elif in_star and key in ("allow", "disallow") and value: - rules.append((key, value)) - except OSError: - rules = [] + for raw in ROBOTS_TXT.splitlines(): + line = raw.split("#", 1)[0].strip() + if not line or ":" not in line: + continue + key, value = (s.strip() for s in line.split(":", 1)) + key = key.lower() + if key == "user-agent": + in_star = value == "*" + elif in_star and key in ("allow", "disallow") and value: + rules.append((key, value)) + # FAIL CLOSED. The first release read the snapshot from a file the wheel + # did not ship, got zero rules and allowed every URL. An empty rule set + # is never a valid reading of avito.ru's robots.txt. + if not any(d == "disallow" for d, _ in rules): + raise RuntimeError("robots snapshot yielded no Disallow rules — refusing " + "to treat every URL as allowed") _ROBOTS = rules return rules @@ -277,11 +277,11 @@ def detect_page_state(html: str, status: Optional[int] = None, blocked a refusal with no puzzle in it (403/451, or nothing at all) unknown served, not yet painted, nothing recognisable - Ordered by how much each signal PROVES (CLAUDE.md §17): content markers + Ordered by how much each signal PROVES (2scraper family rule): content markers first, the wall's own title next, and only then the structural guesses. Deliberately no `captcha` substring test: the Browser API extension injects `chrome-extension://…/captcha/*` scripts into every page it - loads (CLAUDE.md §8, §19). + loads (2scraper family rule). """ html = html or "" kind = page_kind(url) if url else None @@ -341,7 +341,7 @@ def parse_number(text: Any) -> Optional[float]: def count_before_word(text: Optional[str]) -> Optional[int]: """`'1 524 отзыва'` -> 1524. Reads the number BEFORE the word, never every - digit in the string (CLAUDE.md §10: a `review_count` once came out as + digit in the string (2scraper family rule: a `review_count` once came out as 445279961 by stripping every digit out of an aria-label).""" if not text: return None diff --git a/pyproject.toml b/pyproject.toml index 91822d8..de241c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "avito-scraper" -version = "1.0.0" +version = "1.0.1" description = "avito.ru listing, item and seller scraper (Playwright, Selenium, Puppeteer, or the Scraping Browser API over CDP) — GeeTest firewall solving, proxies, fingerprints" readme = "README.md" requires-python = ">=3.9" @@ -35,14 +35,14 @@ selenium = ["selenium>=4.15"] puppeteer = ["pyppeteer>=1.0.2"] [project.scripts] -avito-scraper = "playwright_scraper:main" +avito-scraper = "avito_cli:main" # A flat collection of top-level scripts, so setuptools is told explicitly # rather than asked to guess among this many modules. [tool.setuptools] py-modules = [ - "browser_bridge", "captcha_solver", "diff_runs", "env_config", + "avito_cli", "browser_bridge", "captcha_solver", "diff_runs", "env_config", "fingerprint_client", "output_writer", "page_flow", "playwright_scraper", - "product_parser", "proxy_forwarder", "proxy_pool", "puppeteer_scraper", + "product_parser", "proxy_forwarder", "proxy_pool", "puppeteer_scraper", "robots_snapshot", "selenium_scraper", ] diff --git a/requirements.txt b/requirements.txt index f2f92bb..be3f7a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,21 @@ -# bershka-scraper — core dependencies +# avito-scraper — core dependencies # # pip install -r requirements.txt # -# Needed by everything, including the offline test suite AND -# scraper_api_client.py (the browserless fourth engine -- unlike the three -# browser engines below, it needs no extra requirements-*.txt file: `requests` -# is all it uses). These two are shared by every engine and by diff_runs.py / +# Needed by everything, including the offline test suite, diff_runs.py and # env_config.py. beautifulsoup4>=4.12.0 requests>=2.31.0 -# Then either run scraper_api_client.py directly (see its own docstring -- -# this site was measured 2026-09-10 to need no browser, no proxy and no -# --cdp-url for the plain HTTP path to work), or install exactly ONE of the -# three browser engines for a page kind -# this repo has not yet measured needing a browser for: +# Then install exactly ONE browser engine — avito.ru cannot be fetched without +# a browser (plain HTTP clients get HTTP 429 from its firewall; see README): # # pip install -r requirements-playwright.txt && playwright install chromium # pip install -r requirements-selenium.txt # pip install -r requirements-puppeteer.txt # -# Install one, not all three. playwright and pyppeteer declare mutually -# unsatisfiable pins (pyee <12 vs >=13), and pyppeteer and selenium collide on -# urllib3 (<2.0 vs >=2.0). All three do run side by side in practice, because -# neither library touches the incompatible part -- but `pip check` will -# report the conflict, and pip may resolve it by downgrading something you -# wanted. If you genuinely need more than one engine, use a virtualenv per -# engine (this is exactly what CI's engine-smoke job does). +# One, not all three: playwright and pyppeteer declare mutually unsatisfiable +# pins (pyee), and pyppeteer and selenium collide on urllib3. `pip check` +# reports the conflict and pip may resolve it by downgrading something you +# wanted. Use a virtualenv per engine if you need more than one — exactly what +# CI's engine-smoke job does. diff --git a/robots.snapshot.txt b/robots_snapshot.py similarity index 95% rename from robots.snapshot.txt rename to robots_snapshot.py index 85227fa..b22877e 100644 --- a/robots.snapshot.txt +++ b/robots_snapshot.py @@ -1,4 +1,21 @@ -User-agent: Yandex +""" +robots_snapshot.py +------------------ +avito.ru's robots.txt, as fetched on 2026-10-06 (sha256 dea0300f7e9854b10b96ab5af7a5011277f014e4cf31db3f75c85329c7a05045). + +A Python module rather than a data file on purpose: setuptools ships every +module in `py-modules`, but a loose `robots.snapshot.txt` beside them was +silently LEFT OUT of the wheel, so an installed copy read zero rules and +allowed every URL the checkout refused (found by an external audit, +2026-10-07). product_parser refuses to run on an empty rule set. + +Refresh: replace the string with the live file and update the date and hash. +""" + +FETCHED = "2026-10-06" +SHA256 = "dea0300f7e9854b10b96ab5af7a5011277f014e4cf31db3f75c85329c7a05045" + +ROBOTS_TXT = """User-agent: Yandex Disallow: /auto-logbook Allow: /pets/lost-and-found$ Allow: /pets/lost-and-found/$ @@ -390,3 +407,4 @@ User-agent: YandexDirect Allow: / Sitemap: https://www.avito.ru/sitemap/index.xml +""" diff --git a/selenium_scraper.py b/selenium_scraper.py index 9046e7f..d2abaa2 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -35,7 +35,7 @@ from proxy_forwarder import ProxyForwarder # Module level, so the offline suite can tell "engine absent" from "engine -# broken" (CLAUDE.md §10). +# broken" (2scraper family rule). from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By diff --git a/smoke_test.py b/smoke_test.py index fc5b463..b3c35e5 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -118,7 +118,7 @@ def test_a_wall_is_never_content_even_with_cards_in_it(): def test_extension_markers_are_not_a_captcha(): """The Scraping Browser's extension injects `.../captcha/...` scripts into - every page it loads (CLAUDE.md §8, §19).""" + every page it loads (2scraper family rule).""" import product_parser as P html = fixture("listing_dom.html").replace( "", '