v1.0.1: fixes from the external audit

.dockerignore

@@ -1,9 +1,13 @@
 .git/
 .github/
+# EVERY .env variant: a credential baked into an image is published to
+# everyone who can pull it.
 .env
+.env.*
+*.bak
 # A downloaded proxy list is a credential file, one login and password
 # per line. The Dockerfile COPYs an explicit list rather than `COPY . .`,
-# so this cannot get in today -- but a defence that depends on nobody ever
+# so this cannot get in today — but a defence that depends on nobody ever
 # writing `COPY . .` is not a defence. Same narrow glob as .gitignore.
 proxylist*.txt
 *.log
@@ -16,7 +20,13 @@ __pycache__/
 .venv/
 venv/
 env/
-bershka_products.*
-canary_run.*
+# Scrape output, dumps and research captures.
+live/
+recon/
+captures/
+canary.*
+canary*.meta.json
+*.latest_attempt.meta.json
 *_debug.html
 *_debug.png
+*_p[0-9]*.html

.github/ci_checks.py

@@ -2,14 +2,15 @@
 """
 CI checks that are too long to live inside the workflow YAML.
 
-A workflow step CALLS these; it never re-implements them (family CLAUDE.md
-§11). A heredoc in YAML is a check nobody runs locally and nobody notices
+A workflow step CALLS these; it never re-implements them (a 2scraper family
+rule). A heredoc in YAML is a check nobody runs locally and nobody notices
 going stale. Run any of these from the repo root:
 
     python .github/ci_checks.py --help-check
     python .github/ci_checks.py --sample-check
     python .github/ci_checks.py --secret-check
     python .github/ci_checks.py --run-check --output-prefix live/canary
+    python .github/ci_checks.py --wheel-check
     python .github/ci_checks.py --all
 
 Each prints what it looked at and exits non-zero on failure.
@@ -174,8 +175,73 @@ def run_check(prefix="sample_output"):
     return failed
 
 
+WHEEL_PROBE = r"""
+import sys, product_parser as P, output_writer as O, page_flow
+problems = []
+rules = P._robots_rules()
+if len(rules) < 10:
+    problems.append('robots rules in the installed wheel: %d' % len(rules))
+for url, allowed in (('https://www.avito.ru/moskva?f=x', False),
+                     ('https://www.avito.ru/moskva/telefony?s=104', False),
+                     ('https://www.avito.ru/moskva/telefony?p=2', True)):
+    if P.robots_verdict(url)[0] is not allowed:
+        problems.append('robots verdict for %s is not %s' % (url, allowed))
+if O.SOURCE_DEFAULT != 'avito.ru':
+    problems.append('source is %r' % O.SOURCE_DEFAULT)
+if P.page_url('https://www.avito.ru/moskva/telefony', 3) != 'https://www.avito.ru/moskva/telefony?p=3':
+    problems.append('pagination convention did not survive packaging')
+if not page_flow.should_solve('captcha'):
+    problems.append('state policy did not survive packaging')
+print('\n'.join(problems))
+sys.exit(1 if problems else 0)
+"""
+
+
+def wheel_check(with_playwright=True):
+    """Build the wheel, install it OUTSIDE the checkout, and use it as a user
+    would. v1.0.0 passed an import-only version of this while its wheel
+    shipped no robots rules (every URL allowed) and its console command died
+    on a missing playwright (external audit, 2026-10-07).
+    """
+    import os
+    import tempfile
+    failed = []
+    with tempfile.TemporaryDirectory() as tmp:
+        dist = os.path.join(tmp, "dist")
+        venv = os.path.join(tmp, "venv")
+        step = subprocess.run([sys.executable, "-m", "pip", "wheel", "--no-deps", "-q",
+                               "-w", dist, str(REPO)], capture_output=True, text=True)
+        if step.returncode:
+            return ["wheel build failed: " + step.stderr[-400:]]
+        subprocess.run([sys.executable, "-m", "venv", venv], check=True)
+        py = os.path.join(venv, "bin", "python")
+        cli = os.path.join(venv, "bin", "avito-scraper")
+        wheel = os.path.join(dist, os.listdir(dist)[0])
+        subprocess.run([py, "-m", "pip", "install", "-q", wheel], check=True)
+        probe = subprocess.run([py, "-c", WHEEL_PROBE], capture_output=True, text=True, cwd=tmp)
+        if probe.returncode:
+            failed += ["installed wheel: " + l for l in probe.stdout.splitlines() + [probe.stderr[-300:]] if l]
+        else:
+            print("ok       installed wheel: robots rules, verdicts, conventions")
+        bare = subprocess.run([cli, "--help"], capture_output=True, text=True, cwd=tmp)
+        if bare.returncode != 2 or "avito-scraper[playwright]" not in bare.stderr:
+            failed.append("console script without playwright: rc=%s, %r"
+                          % (bare.returncode, (bare.stderr or bare.stdout)[-200:]))
+        else:
+            print("ok       avito-scraper without playwright: says what to install (exit 2)")
+        if with_playwright:
+            subprocess.run([py, "-m", "pip", "install", "-q", wheel + "[playwright]"], check=True)
+            full = subprocess.run([cli, "--help"], capture_output=True, text=True, cwd=tmp)
+            if full.returncode != 0 or "--mode" not in full.stdout:
+                failed.append("console script with playwright: rc=%s, %r"
+                              % (full.returncode, full.stderr[-200:]))
+            else:
+                print("ok       avito-scraper[playwright]: --help runs")
+    return failed
+
+
 CHECKS = {"help": help_check, "sample": sample_check, "secret": secret_check,
-          "run": run_check}
+          "run": run_check, "wheel": wheel_check}
 
 
 def main():
@@ -193,12 +259,14 @@ def main():
                              "(use with --output-prefix)")
     parser.add_argument("--secret-check", action="store_true",
                         help="No credentials committed anywhere")
+    parser.add_argument("--wheel-check", action="store_true",
+                        help="Build the wheel, install it outside the checkout, use it")
     parser.add_argument("--all", action="store_true",
-                        help="help + sample + secret (the run check needs a live run)")
+                        help="help + sample + secret (run needs a live run; wheel is slow)")
     args = parser.parse_args()
 
     selected = [name for name in CHECKS
-                if (args.all and name != "run") or getattr(args, f"{name}_check")]
+                if (args.all and name not in ("run", "wheel")) or getattr(args, f"{name}_check")]
     if not selected:
         parser.error("pick at least one check, or --all")
     failures = []

.github/workflows/canary.yml

@@ -43,7 +43,8 @@ jobs:
       - name: Explain the skip
         if: env.READY != 'true'
         run: |
-          echo "::notice::Skipped: needs secrets TWOCAPTCHA_KEY and AVITO_PROXY and the"
+          echo "::warning::LIVE RUN SKIPPED — this green run proves the workflow is healthy, NOT that avito.ru was scraped."
+          echo "::notice::Needs secrets TWOCAPTCHA_KEY and AVITO_PROXY and the"
           echo "::notice::variable AVITO_BROWSER_ACCOUNT_ID. A GitHub runner is not a"
           echo "::notice::Russian residential address; a live run from here without the"
           echo "::notice::Scraping Browser would be red every day regardless of the code."
@@ -87,7 +88,7 @@ jobs:
 
       - name: Assert the data, not just the exit code
         if: env.READY == 'true'
-        # THE SHIPPED CHECK, not an inline copy (CLAUDE.md §11).
+        # THE SHIPPED CHECK, not an inline copy (a 2scraper family rule).
         run: |
           echo "exit code from the run: $rc"
           python3 .github/ci_checks.py --run-check --sample-check --output-prefix canary

.github/workflows/claude-code-review.yml

@@ -77,17 +77,20 @@ jobs:
             you can find is a number or a claim that no longer matches the
             code, or a comment that explains a reason the code no longer has.
 
-            A few things specific to this repo worth checking: every price is
-            in EUR and every row is a DOM reading (there is no JSON-LD on
-            this site to cross-check against — see output_writer.py and
-            diff_runs.py); there are two row schemas, Player and Transfer
-            (output_writer.py), and a change to one should not silently affect
-            the other; `page` and `row_index` must stay unique across a
-            paginated run (see product_parser.py and the tokopedia lesson in
-            CLAUDE.md); and this site's listing pagination is robots-DISALLOWED,
-            so product_parser.page_url() returning its input unchanged is a
-            deliberate decision, not a stub — a change that makes it
-            construct a URL deserves extra scrutiny.
+            A few things specific to this repo (avito.ru) worth checking:
+            `price` must stay the DISPLAYED price — Avito's bare price fields
+            are the pre-discount figure (product_parser.py docstring); a
+            listing page arrives in two renderings, DOM and an embedded JSON
+            state, and `price_source` says which, so diff_runs.py must not
+            report a rendering change as a price change; there are three row
+            schemas, Product / Item / Seller (output_writer.py), and Item must
+            keep Product's column prefix; `page` and `position` must stay
+            unique across a paginated run; a captcha solve must go through the
+            browser's own proxy (captcha_solver.py — a token solved from
+            another IP is rejected); a transport fault is exit 5, never 3;
+            phone numbers, IMEI and a private seller's placeholder name must
+            never reach a row; and a URL robots_snapshot.py disallows must be
+            refused before any request.
 
             Also worth flagging: a check that cannot fail, an error path that
             swallows a failure silently, a credential that could reach a

.github/workflows/tests.yml

@@ -43,7 +43,7 @@ jobs:
           pytest -q
 
       - name: Every CLI answers --help
-        # THE SHIPPED CHECK, not a copy of it (CLAUDE.md §11). Engine
+        # THE SHIPPED CHECK, not a copy of it (a 2scraper family rule). Engine
         # libraries are not installed here, so those entry points report
         # skipped rather than failed.
         run: python3 .github/ci_checks.py --help-check
@@ -55,29 +55,13 @@ jobs:
         run: python env_config.py
 
       - name: The built wheel installs and works outside the checkout
-        # A flat `py-modules` collection is easy to get wrong, and a wheel
-        # that imports in the source tree but not once installed fails only
-        # on delivery. Installed into its own venv, exercised from /tmp.
-        run: |
-          python -m pip install --quiet build
-          python -m build --wheel
-          python -m venv /tmp/wheelcheck
-          /tmp/wheelcheck/bin/pip install --quiet dist/*.whl
-          cd /tmp
-          /tmp/wheelcheck/bin/python -c "
-          import sys, output_writer as O, page_flow, product_parser as P, captcha_solver, proxy_forwarder
-          problems = []
-          if O.SOURCE_DEFAULT != 'avito.ru':
-              problems.append('source is %r' % O.SOURCE_DEFAULT)
-          u = 'https://www.avito.ru/moskva/telefony'
-          if P.page_url(u, 3) != u + '?p=3':
-              problems.append('pagination convention did not survive packaging')
-          if not page_flow.should_solve('captcha'):
-              problems.append('state policy did not survive packaging')
-          for p in problems:
-              print('::error::' + p)
-          print('installed wheel: %d problem(s)' % len(problems))
-          sys.exit(1 if problems else 0)"
+        # THE SHIPPED CHECK. Builds the wheel, installs it into a fresh venv
+        # outside the checkout and uses it: robots rules present and
+        # refusing, the console command without playwright says what to
+        # install, and with it answers --help. v1.0.0 passed an import-only
+        # inline version of this while its wheel allowed every URL and its
+        # command died on a missing playwright (external audit, 2026-10-07).
+        run: python3 .github/ci_checks.py --wheel-check
 
       - name: No credentials committed
         # THE SHIPPED CHECK — one scanner, shared with the git hooks (§21).
@@ -90,7 +74,7 @@ jobs:
     #
     # ONE VENV PER ENGINE: the engines' pins are mutually unsatisfiable
     # (pyee, urllib3), and installed together pip may resolve the conflict by
-    # picking a release nobody uses (CLAUDE.md §11).
+    # picking a release nobody uses (a 2scraper family rule).
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -168,7 +152,8 @@ jobs:
           import importlib, sys
           mods = ['browser_bridge','captcha_solver','diff_runs','env_config',
                   'fingerprint_client','output_writer','page_flow',
-                  'playwright_scraper','product_parser','proxy_forwarder','proxy_pool']
+                  'playwright_scraper','product_parser','proxy_forwarder','proxy_pool',
+                  'robots_snapshot']
           bad = []
           for m in mods:
               try: