pyppeteer survives a failed load without a proxy; a late challenge can be solved

.env.example

@@ -74,4 +74,6 @@ KOSE_CDP_ENDPOINT=ws://{login}-zone-scraping_browser-country-jp-pid-{profileId}:
 KOSE_PROXY=http://{user}:{password}@jp.proxy.2captcha.com:2334
 
 # A default URL, so a run needs no --url. An explicit flag still wins.
-KOSE_URL=https://maison.kose.co.jp/site/cosmedecorte/c/c15/
+# Shipped commented out: a copied .env should not decide what a run fetches,
+# so --tags and --category keep working until you choose a default here.
+# KOSE_URL=https://maison.kose.co.jp/site/cosmedecorte/c/c15/

.github/ci_checks.py

@@ -86,14 +86,26 @@
 HEX32 = re.compile(r"\b[0-9a-f]{32}\b")
 # Contexts in which a 32-hex string is plainly not a key. Site-specific
 # entries belong here only once a committed fixture actually carries one —
-# CLAUDE.md §16: a family-wide entry copied without checking is how dead
+# A family-wide entry copied without checking is how dead
 # code spreads. Kept as a CONTEXT allowlist rather than by loosening the
 # pattern: a bare 32-hex string anywhere else still fails, which is the
 # point of the check.
 HEX32_ALLOWED = ("sha", "hash", "nonce", "example", "md5", "digest",
                  "checksum")
 
-SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example")
+SCANNED_SUFFIXES = (".py", ".md", ".txt", ".yml", ".yaml", ".example",
+                    ".json", ".csv", ".toml", ".cfg", ".ini")
+
+
+def _is_scanned(path):
+    """A file of a scanned type, or any `.env*` file whatever its suffix.
+
+    A working copy of `.env` renamed the way people rename them — `.env.bak`,
+    `.env.local`, `.env.save` — is untracked, ignored by git, and exactly
+    where a live key sits. Reading it here is what lets a local run of this
+    check say so before anything is committed.
+    """
+    return path.suffix in SCANNED_SUFFIXES or path.name.startswith(".env")
 
 
 # Directory names that are never this repo's own source.
@@ -112,7 +124,7 @@ def _is_inside_a_virtualenv(path):
     `.v` to fail, on a 32-hex string inside pip's vendored
     `packaging/_elffile.py`.
 
-    That is CLAUDE.md §21's failure exactly — a check that fails on a
+    That is the failure exactly — a check that fails on a
     legitimate tree is a check nobody can read, and it fails for everyone but
     the person who chose the blessed name.
     """
@@ -121,14 +133,18 @@ def _is_inside_a_virtualenv(path):
             break
         if (parent / "pyvenv.cfg").exists():
             return True
-    return "site-packages" in path.parts
+    return "site-packages" in path.relative_to(REPO).parts
 
 
 def scanned_files():
     for path in sorted(REPO.rglob("*")):
-        if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
+        if not path.is_file() or not _is_scanned(path):
             continue
-        if any(part in _SKIP_NAMES for part in path.parts):
+        # RELATIVE to the repository. `path.parts` is absolute, so a clone
+        # that merely lives under a directory called `build` or `dist`
+        # skipped every file and reported "0 files scanned, nothing
+        # credential-shaped" — a clean bill for a scan that read nothing.
+        if any(part in _SKIP_NAMES for part in path.relative_to(REPO).parts):
             continue
         if _is_inside_a_virtualenv(path):
             continue
@@ -194,9 +210,9 @@ def sample_check():
         failed.append("sample_output.csv header differs from output_writer.Product")
 
     if not failed:
-        discounted = sum(1 for r in rows if r.get("original_price"))
+        priced = sum(1 for r in rows if r.get("price") is not None)
         print(f"ok       {len(rows)} rows, {len(expected)} columns, "
-              f"{discounted} discounted, schema matches")
+              f"{priced} priced, schema matches")
     return failed
 
 
@@ -220,6 +236,10 @@ def secret_check():
                 failed.append(f"{rel}:{lineno} contains {match[:6]}… — a "
                               f"32-char hex string, the shape of a 2captcha key")
 
+    if not scanned:
+        # Zero files is never a clean result: it means the scan did not run.
+        failed.append(f"no files were scanned under {REPO} — the scan read "
+                      f"nothing, so it cannot vouch for anything")
     if not failed:
         print(f"ok       {scanned} files scanned, nothing credential-shaped")
     return failed

.github/workflows/canary.yml

@@ -10,16 +10,15 @@
 # all, with no challenge of any kind. A GitHub runner is a datacentre address
 # too, so this workflow re-measures that claim every morning.
 #
-# CLAUDE.md §11 says a canary that CANNOT pass must skip rather than fail,
-# because a check that is always red teaches everyone to ignore checks. The
-# other half of that rule applies here: **a canary that CAN pass without a
-# credential must never be gated on one.** If Kosé ever puts its catalogue
-# behind a key or a bot manager, this badge goes red the next morning and the
-# README's claim is retested without anyone remembering to.
+# **A canary that CAN pass without a credential must never be gated on one.**
+# If Kosé ever puts its catalogue behind a key or a bot manager, this badge
+# goes red the next morning and the README's claim is retested without anyone
+# remembering to.
 #
-# The last job is the exception and it is guarded: it exercises the PAID path
-# (the Scraping Browser API over CDP), which cannot run without a secret, so
-# it skips with a ::notice:: when none is set.
+# The last job exercises the PAID path (the Scraping Browser API over CDP),
+# which cannot run without a secret. It does not pass by skipping: with no
+# secret it FAILS with an ::error:: naming the secret, because a green check
+# that tested nothing reads exactly like one that tested something.
 #
 #     gh secret set KOSE_CDP_ENDPOINT --repo 2scraper/kose-scraper
 #
@@ -40,8 +39,8 @@ permissions:
 
 env:
   # Cosme Decorté's catalogue: 13 pages of 24 on 2026-09-18, so three pages
-  # really is three pages. §11: with one page pagination is never exercised,
-  # which is exactly how a silent single-page bug survived in a sibling repo.
+  # really is three pages. With one page pagination is never exercised, which
+  # is exactly how a silent single-page bug survived in a sibling repo.
   CANARY_URL: "https://maison.kose.co.jp/site/cosmedecorte/c/c15/"
   CANARY_PRODUCT: "https://maison.kose.co.jp/site/onebykose/g/gMUSC/"
 
@@ -105,7 +104,7 @@ jobs:
               if not ok:
                   failures.append(f"{label}: {detail}")
 
-          # §20: fail on a parser failure BY NAME. "0 products" would send
+          # Fail on a parser failure BY NAME. "0 products" would send
           # whoever reads this red badge to check the URL, when the thing that
           # moved is the parser. This stop_reason means the site served a page
           # that links to products and we turned it into nothing.
@@ -147,7 +146,7 @@ jobs:
               want(f"{column} coverage", share >= 95.0,
                    f"{share:.0f}% ({filled}/{len(rows)})")
 
-          # Values, not just coverage (§10). A column can be 100% populated
+          # Values, not just coverage. A column can be 100% populated
           # and entirely wrong.
           want("every currency is JPY",
                {r.get("currency") for r in rows if r.get("price") is not None} == {"JPY"},
@@ -164,7 +163,7 @@ jobs:
                all(r["sku"] and f"/g/g{r['sku']}/" in r["url"] for r in rows),
                "a row points at a product other than the one it describes")
 
-          # §18: page + position must be unique across a multi-page run, or
+          # page + position must be unique across a multi-page run, or
           # `position` is worthless — it restarts at 1 on every page.
           pairs = [(r.get("page"), r.get("position")) for r in rows]
           want("page+position unique", len(set(pairs)) == len(pairs),
@@ -377,25 +376,23 @@ jobs:
           if-no-files-found: ignore
 
   scraping_browser:
-    name: Scraping Browser API (skips without a secret)
+    name: Scraping Browser API (needs the KOSE_CDP_ENDPOINT secret)
     runs-on: ubuntu-latest
     timeout-minutes: 20
     env:
       # `secrets` is not available in a job-level `if`, so presence is tested
       # here and read in each step's `if`.
       HAVE_CDP: ${{ secrets.KOSE_CDP_ENDPOINT != '' }}
     steps:
-      - name: Explain the skip
+      - name: Require the secret
         if: env.HAVE_CDP != 'true'
         run: |
-          echo "::notice::Skipping the Scraping Browser run: no KOSE_CDP_ENDPOINT secret is set."
-          echo "This path is OPTIONAL on this site — the listing job above runs"
-          echo "the same scrape from a bare runner with no credentials at all,"
-          echo "which is the README's central claim. To exercise the paid path:"
+          echo "::error::The KOSE_CDP_ENDPOINT secret is not set, so the Scraping Browser path was NOT tested."
+          echo "The listing jobs above test the site itself with no credentials."
+          echo "This job tests the paid path and fails rather than passing on a"
+          echo "skip. To make it run:"
           echo "    gh secret set KOSE_CDP_ENDPOINT --repo 2scraper/kose-scraper"
-          echo "Note a Scraping Browser profile's credentials live about a day,"
-          echo "so a stored secret goes stale between scheduled runs — prefer"
-          echo "dispatching this workflow manually when you want to test it."
+          exit 1
 
       - uses: actions/checkout@v4
         if: env.HAVE_CDP == 'true'
@@ -415,7 +412,7 @@ jobs:
         env:
           # Through the ENVIRONMENT, never a command line: env_config.py reads
           # exactly this variable, and a credential in argv is readable by
-          # anything that can run `ps` (CLAUDE.md §3, §11).
+          # anything that can run `ps`.
           KOSE_CDP_ENDPOINT: ${{ secrets.KOSE_CDP_ENDPOINT }}
         run: |
           set +e
@@ -432,11 +429,11 @@ jobs:
           import json, os, sys
           code = int(os.environ["EXIT_CODE"])
           if code == 5:
-              print("::warning::Remote API error (exit 5). A Scraping Browser "
-                    "profile allows ONE live connection and its credentials "
-                    "live about a day, so a stale secret or a busy pid looks "
-                    "exactly like this. Not a failure of the scraper.")
-              sys.exit(0)
+              print("::error::Exit 5: the content was never obtained over CDP. "
+                    "A Scraping Browser profile allows ONE live connection, so "
+                    "a busy pid looks like this; an expired profile answers "
+                    "401. Either way nothing was tested.")
+              sys.exit(1)
           if code != 0:
               print(f"::error::The CDP run exited {code}.")
               sys.exit(1)

.github/workflows/tests.yml

@@ -51,7 +51,7 @@ jobs:
 
       - name: Every CLI answers --help, and the sample output is real
         # BOTH of these were once INLINE reimplementations of checks that
-        # `.github/ci_checks.py` already implements — which is CLAUDE.md §17's
+        # `.github/ci_checks.py` already implements — which is the
         # "two sources of truth, one dead and one holed", and it bit on this
         # repo's very first CI run: the inline sample check still imported
         # `output_writer.Business`, a class this repo renamed, so the job died

.gitignore

@@ -24,6 +24,23 @@ live_results.zip
 *_debug.html
 *_debug.png
 *.html.page*
+# `--dump-html NAME --pages N` writes NAME.pageN whatever NAME ends in, and a
+# one-page dump named `dump.html` lands at the root.
+*.page[0-9]*
+/*.html
+
+# --- working directories by KIND, not by one chosen name -----------------
+live/
+out/
+output/
+run/
+runs/
+result/
+results/
+tmp/
+scratch/
+dump/
+dumps/
 
 # --- captures kept for parser work ---------------------------------------
 # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py);
@@ -49,7 +66,7 @@ build/
 
 # --- environments --------------------------------------------------------
 # `.venv*` rather than `.venv/`, because the README tells you to make ONE
-# VENV PER ENGINE (§6: the three engines' pins are mutually unsatisfiable),
+# VENV PER ENGINE (the three engines' pins are mutually unsatisfiable),
 # so `.venv-playwright/` and `.venv-selenium/` are the normal names here and
 # the bare `.venv/` pattern misses every one of them.
 .venv*
@@ -77,7 +94,7 @@ secrets*
 # recoverable only because that repo was private at the time. On 2026-09-17
 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC
 # repository, one `git add -A` from publication. Craigslist was fixed then
-# and the fix never reached anyone else, which is §16's rule unapplied: when
-# you fix one, check the siblings.
+# and the fix never reached anyone else: when you fix one, check the
+# siblings.
 .env*
 !.env.example

CHANGELOG.md

@@ -8,6 +8,55 @@ as a CLI toolkit can. A **patch** release means fixes; it does not promise
 that every flag and default is frozen. Where a patch changes behaviour an
 existing user would notice, the release notes lead with it.
 
+## [Unreleased]
+
+> **Newly refused flag values.** `--pages 0`, `--concurrency 0` and any
+> negative `--pages`, `--concurrency`, `--delay`, `--retry-delay` or
+> `--proxy-block-retries` are now usage errors (exit 2) naming the flag, in
+> every engine and in `scraper_api_client.py`. They used to be accepted: a
+> zero page count fetched nothing and reported an empty catalogue, and a
+> negative pause crashed the run after page 1.
+>
+> **pyppeteer, a page that never loads, no `--proxy`:** the run now ends as a
+> load failure (exit 5, or 6 with rows) instead of a traceback and exit 1.
+> With a pool, a repeated timeout no longer rotates the exit; only an
+> unusable exit does, as in the other engines.
+
+### Fixed
+
+- **pyppeteer crashed with `AttributeError` when a page failed to load and
+  no proxy was configured.** The load-failure branch called
+  `pool.advance(...)` on a `None` pool. It now rotates only for an unusable
+  exit, and only when the pool has another exit to rotate to.
+- **The per-page solve allowance was spent by a detection that bought
+  nothing.** The pre-classification captcha check used up the page's one
+  solve even when it found no challenge, so a challenge first seen after
+  classification was never offered to the solver and the page was reported
+  as blocked. The allowance is now claimed immediately before the paid
+  request, in all three browser engines. A purchase that fails still counts,
+  so the cap still holds at one per page across retries and rotations.
+- **The credential scan read nothing from a clone under a directory named
+  `build` or `dist`.** The skip list was tested against the absolute path. It
+  now tests the path relative to the repository, a scan that reads zero
+  files fails, and `.env*` backups, `.json` and `.csv` files are read too.
+- `.gitignore` now covers every `--dump-html NAME --pages N` output
+  (`NAME.pageN`), a root-level `*.html` dump and the usual working
+  directories (`live/`, `out/`, `tmp/`, ...).
+- `.env.example` ships `KOSE_URL` commented out, so a copied `.env` does not
+  decide what a run fetches.
+- The canary's Scraping Browser job no longer passes by skipping: with no
+  `KOSE_CDP_ENDPOINT` secret it fails with an error naming the secret, and an
+  exit 5 over CDP is a failure rather than a warning.
+- Comments that named columns this schema does not have, stated a category
+  total as current, carried another vendor's wording or gave the wrong exit
+  code for `parser_found_nothing` (it is 5) were corrected; the CI sample
+  summary counts priced rows instead of a column that does not exist.
+- Shipped files, including the README, `--help` output and the 0.1.1 notes
+  below, no longer cite private notes by name or section number; a suite
+  check keeps it that way.
+- `smoke_test.py -v` no longer crashes on a check whose detail is not a
+  string.
+
 ## [0.2.0] — 2026-10-13
 
 Worked through a third-party audit. Its headline conclusion was wrong and
@@ -190,7 +239,7 @@ as if they were this one.
 
 ## [0.1.1] — 2026-09-18
 
-A pass back over CLAUDE.md before calling the repo finished. Everything here
+A pass back over the project's own rules before calling the repo finished. Everything here
 was found by checking a claim rather than by reading the code.
 
 ### Fixed

CONTRIBUTING.md

@@ -96,9 +96,11 @@ Then the rest of the presentation, in the order that matters:
 1. `python3 smoke_test.py` green, and the canary dispatched at least once.
    **Both of this repo's real canary jobs run with NO secrets** and are
    expected to be green — that is not a convenience, it is what keeps the
-   README's central claim honest. Only the third job, which exercises the
-   Scraping Browser path, skips without a secret; dispatch it by hand once
-   and confirm the SKIP branch runs, not just the happy one.
+   README's central claim honest. Only the last job, which exercises the
+   Scraping Browser path, needs a secret (`KOSE_CDP_ENDPOINT`); without it
+   that job FAILS with an error naming the secret rather than passing on a
+   skip, because a green check that tested nothing is indistinguishable
+   from one that tested something.
    Note `workflow_dispatch` requires the workflow to exist on the DEFAULT
    branch — from a feature branch `gh workflow run` answers
    `HTTP 404: workflow canary.yml not found on the default branch`, which
@@ -143,7 +145,7 @@ that breaks one will fail rather than silently regress:
   (種類を選ぶ, カートに入れる, 商品を購入する, 予約受付中) and two mean not
   (現在購入頂けません, 予約受付終了). A wording Kosé adds tomorrow must read as
   unknown rather than as a quiet "in stock". The column is verified in BOTH
-  directions — 19 of 171 tiles were unavailable — which §20 says is otherwise
+  directions — 19 of 171 tiles were unavailable — which is otherwise
   not verified at all.
 
 - **`tax_rate` is a column because Japan has two rates.** 10% standard, 8%

README.md

@@ -339,8 +339,7 @@ into 264. Every fixture in the suite pins the value, not the coverage.
 
 ## What has and has not been verified live
 
-Honest about the gap, because §13 of this family's playbook says a claim is
-measured or absent:
+Honest about the gap, because a claim here is either measured or absent:
 
 | path | status, 2026-10-13 |
 |---|---|

captcha_solver.py

@@ -614,8 +614,8 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st
                      api_version: str = "v2", min_score: float = 0.7) -> str:
     """Public entry point: solve `challenge` through 2captcha and return the token.
 
-    There used to be a `use_antidetect` branch here, behind a CLI flag of the
-    same name, that POSTed to a hardcoded local "solve-captcha" endpoint.
+    There used to be a second branch here, behind a CLI flag of its own,
+    that POSTed to a hardcoded local "solve-captcha" endpoint.
     It was removed before publishing: that endpoint was a placeholder for a
     product that does not exist under that name, so the flag could not work for
     anyone who set it, and the real "the browser solves it for you" path is
@@ -715,4 +715,4 @@ class CaptchaUnsolvable(RuntimeError):
 # what the address has been doing — a narrow list is how a challenge gets
 # reported as an empty page months later. A solver for a challenge this site
 # has never been observed to serve is dead code; a DETECTOR for one is cheap
-# insurance. See the family note in CLAUDE.md.
+# insurance.