From 19ac640db392c905cd749491eaa13ad02855746c Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:58:10 +0000 Subject: [PATCH] Selenium and pyppeteer no longer crash on a no-proxy block, read --mode job pages, and CSV cells are formula-safe - Selenium/pyppeteer: the block retry without a proxy pool re-fetches through the same path instead of raising AttributeError; pyppeteer rotates on a load failure only for a dead exit with a pool. - Selenium/pyppeteer pass --mode to every classify call, so job pages are read as jobs instead of parse errors. - Playwright/pyppeteer pass the HTTP status to the classifier, so a 404 exits 4; the Scraper API client handles not_found the same way. - CSV cells starting with = + - @ TAB CR LF get a leading apostrophe (text only, CSV only); the sidecar records csv_cells_escaped. - .env.example ships WELLFOUND_URL commented out, and an environment URL yields to --role/--location/--remote and an explicit --mode job/jobs. - Counting flags reject values that skipped the fetch or crashed (--pages, --concurrency, --proxy-block-retries, --delay, --retry-delay; --timeout/--retries/--retry-delay in the Scraper API client). - Credential scan exclusions are judged inside the repository, and a scan of 0 files fails. - Canary fails with an error naming WELLFOUND_PROXY when it is missing, and a bare-runner job requires the datacentre refusal to be recognised. - .gitignore covers --dump-html page files, live/ and root-level HTML. - Comments and help no longer describe other sites, a nonexistent challenge state or mode, or cite private notes; a suite check guards it. Co-Authored-By: Claude Opus 5.5 --- .env.example | 2 +- .github/ci_checks.py | 14 +- .github/workflows/canary.yml | 121 ++++++-- .github/workflows/tests.yml | 2 +- .gitignore | 9 +- CHANGELOG.md | 57 ++++ CONTRIBUTING.md | 14 +- captcha_solver.py | 26 +- diff_runs.py | 6 +- env_config.py | 2 +- fingerprint_client.py | 2 +- output_writer.py | 80 ++++- page_flow.py | 57 +++- playwright_scraper.py | 124 +++++--- product_parser.py | 63 ++-- puppeteer_scraper.py | 130 +++++--- scraper_api_client.py | 29 +- selenium_scraper.py | 103 ++++--- smoke_test.py | 576 +++++++++++++++++++++++++++++++---- 19 files changed, 1105 insertions(+), 312 deletions(-) diff --git a/.env.example b/.env.example index ca4b876..9471e66 100644 --- a/.env.example +++ b/.env.example @@ -139,7 +139,7 @@ WELLFOUND_PROXY=http://{user}:{password}@na.proxy.2captcha.com:2334 # residential exit, and this scraper has no mode for it. # # `--role` and `--location` build a landing URL for you, so this is optional. -WELLFOUND_URL=https://wellfound.com/role/r/software-engineer +# WELLFOUND_URL=https://wellfound.com/role/r/software-engineer # Deliberately NOT here: an output prefix. `--out` already carries a # non-empty default, so the loader — which only fills UNSET values — would diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 87d00b0..e47eef1 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -81,7 +81,7 @@ HEX32 = re.compile(r"\b[0-9a-f]{32}\b") # Contexts in which a 32-hex string is plainly not a key. Site-specific # entries belong here only once a committed fixture actually carries one — -# CLAUDE.md §16: a family-wide entry copied without checking is how dead +# A family-wide entry copied without checking is how dead # code spreads. Kept as a CONTEXT allowlist rather than by loosening the # pattern: a bare 32-hex string anywhere else still fails, which is the # point of the check. @@ -107,7 +107,7 @@ def _is_virtualenv(path): walked into pip's vendored code and flagged a 32-hex string in `_elffile.py` as key-shaped. The run was correct about the string and wrong about the file, and a guard people have to argue with is one they - learn to suppress (CLAUDE.md §22). + learn to suppress. Structural rather than by name, for the same reason a parser anchors on a URL pattern instead of a CSS class: `venv`, `.venv`, `.v`, `env39` and @@ -128,7 +128,11 @@ def scanned_files(): continue if path.suffix not in SCANNED_SUFFIXES: continue - if any(part in _SKIP_NAMES for part in path.parts): + # Judged on the path RELATIVE to the repository. The absolute path + # also holds the directories the clone happens to live under, and a + # checkout under ~/build, /tmp/venv or any other skipped name used to + # scan 0 files and report clean. + if any(part in _SKIP_NAMES for part in path.relative_to(REPO).parts): continue if any(root in path.parents for root in skip_roots): continue @@ -220,6 +224,10 @@ def secret_check(): failed.append(f"{rel}:{lineno} contains {match[:6]}… — a " f"32-char hex string, the shape of a 2captcha key") + if not scanned: + # A scan that read nothing has checked nothing; it must not pass. + failed.append(f"0 files scanned under {REPO} — the exclusions skipped " + f"the whole tree, so nothing was checked") if not failed: print(f"ok {scanned} files scanned, nothing credential-shaped") return failed diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 9aea06f..a138f9d 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -1,7 +1,7 @@ # One real run against wellfound.com. tests.yml (deliberately) never touches # the site — see its own header. # -# THIS CANARY IS GATED, AND THAT IS A MEASUREMENT RATHER THAN CAUTION. +# THIS CANARY NEEDS A RESIDENTIAL EXIT, AND THAT IS A MEASUREMENT. # # Wellfound refuses a datacenter address on EVERY route, `robots.txt` # included. Measured 2026-09-17 from a Hetzner VPS: HTTP 403 to plain HTTP, @@ -9,22 +9,29 @@ # residential US exit, HTTP 200 on `/`, `/jobs`, every `/role/…` shape and # `/jobs/{id}-{slug}`, three times out of three. # -# A GitHub runner is a datacenter address. So an ungated live run here would -# be red every single morning, and a check that is always red teaches -# everyone to ignore checks (CLAUDE.md §11). This workflow therefore SKIPS -# with a ::notice:: when no proxy secret is set, and runs for real when one -# is: +# A GitHub runner is a datacenter address, so the job that checks real data +# (`listing`) runs through the WELLFOUND_PROXY secret: # # gh secret set WELLFOUND_PROXY --repo 2scraper/wellfound-scraper # +# WITHOUT that secret the job FAILS with an error naming it. It used to skip +# with a notice and end green, and 28 green runs in a row had never touched +# the site: a skipped canary is not a tested success. +# # The secret goes through the ENVIRONMENT, never a `--proxy` flag: a -# credential on a command line is readable by anything that can run `ps` -# (CLAUDE.md §3). `env_config.py` already reads exactly this variable, and an -# unset secret arrives as an empty string which the loader treats as unset — -# so the same step is correct with and without it. +# credential on a command line is readable by anything that can run `ps`. +# `env_config.py` already reads exactly this variable. +# +# The second job (`datacentre`) needs no secret. It is documented as +# "datacentre refusal expected": it runs the same engine from the bare +# runner and requires the refusal to be RECOGNISED as one (exit 3 with the +# refusal page saved), or real rows if the site ever serves the runner. A +# crash, a timeout or an unreadable page (exit 1/5) is red there. It proves +# the engine reaches the site and classifies what it gets; it does not +# replace `listing`, which is the job that tests real data. # # `secrets` is not available in a job-level `if`, so presence is tested -# through job `env` and read in each step's `if`. +# through job `env` and read in a step. name: canary @@ -40,7 +47,7 @@ permissions: jobs: listing: - name: role listing (needs a residential proxy secret) + name: role listing (needs the WELLFOUND_PROXY secret) runs-on: ubuntu-latest timeout-minutes: 25 env: @@ -48,29 +55,22 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Explain the skip + - name: Require the proxy secret if: env.HAVE_PROXY != 'true' run: | - echo "::notice::Skipping the live run: no WELLFOUND_PROXY secret is set." - echo "Wellfound answers a datacenter address with HTTP 403 on every route," - echo "and a GitHub runner is a datacenter address, so this cannot pass" - echo "without a residential exit. The offline suite in tests.yml covers the" - echo "parser; this job covers the live site only." - echo "" - echo " gh secret set WELLFOUND_PROXY --repo 2scraper/wellfound-scraper" + echo "::error::WELLFOUND_PROXY is not set, so this canary cannot reach wellfound.com and has tested nothing. Wellfound answers a datacenter address (a GitHub runner) with HTTP 403 on every route; set a residential exit: gh secret set WELLFOUND_PROXY --repo 2scraper/wellfound-scraper" { - echo "### Canary skipped" - echo "No \`WELLFOUND_PROXY\` secret. See the job log for why a datacenter" - echo "address cannot reach this site." + echo "### Canary did not run" + echo "No \`WELLFOUND_PROXY\` secret. A datacenter address cannot reach this" + echo "site, so without a residential exit there is nothing to test." } >> "$GITHUB_STEP_SUMMARY" + exit 1 - uses: actions/setup-python@v5 - if: env.HAVE_PROXY == 'true' with: python-version: "3.12" - name: Install - if: env.HAVE_PROXY == 'true' run: | python -m pip install --upgrade pip pip install -r requirements.txt -r requirements-playwright.txt @@ -78,10 +78,9 @@ jobs: # >= 3 pages, because with one page pagination is never exercised — # which is exactly how a silent single-page bug survived in a sibling - # repo (CLAUDE.md §11). + # repo. - name: Run id: run - if: env.HAVE_PROXY == 'true' env: # Through the environment, not argv. See the header. WELLFOUND_PROXY: ${{ secrets.WELLFOUND_PROXY }} @@ -94,7 +93,6 @@ jobs: echo "exit_code=$?" >> "$GITHUB_OUTPUT" - name: Check what came back - if: env.HAVE_PROXY == 'true' env: EXIT_CODE: ${{ steps.run.outputs.exit_code }} run: | @@ -114,7 +112,9 @@ jobs: "credential before concluding the site changed.", 4: "Zero jobs (exit 4). Either this role genuinely has no " "listings now, or the Apollo payload shape moved.", - 5: "Remote API error (exit 5) — nothing was tested.", + 5: "Content not obtained (exit 5): a load timeout, a dead " + "exit, or a page that was served but could not be read " + "(parse_error). Check the saved debug page.", 6: "Partial (exit 6): some pages came back and some did not.", }.get(code, "See the log.")) sys.exit(1) @@ -225,3 +225,66 @@ jobs: canary.meta.json canary_page*_debug.html if-no-files-found: ignore + + datacentre: + # Datacentre refusal expected — see the header. Exit 3 is accepted here + # ONLY because `listing` is the job that requires real rows. + name: bare runner (datacentre refusal expected) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-playwright.txt + playwright install --with-deps chromium + + - name: Run from the runner's own address + id: run + run: | + set +e + python playwright_scraper.py \ + --role software-engineer --remote \ + --pages 1 --format json --out bare + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + + - name: Check the refusal was recognised + env: + EXIT_CODE: ${{ steps.run.outputs.exit_code }} + run: | + code="$EXIT_CODE" + if [ "$code" = "3" ]; then + if ls bare_page1_debug.html >/dev/null 2>&1; then + echo "::notice::Datacentre refusal recognised (exit 3), as expected from a GitHub runner. Real data is checked by the 'listing' job." + exit 0 + fi + echo "::error::Exit 3 without the refusal page saved — the block path did not run as designed." + exit 1 + fi + if [ "$code" = "0" ]; then + rows=$(python -c "import json; print(len(json.load(open('bare.json'))))") + if [ "$rows" -ge 20 ]; then + echo "::notice::Wellfound served the bare runner ($rows rows). The datacentre gating may have changed; re-measure." + exit 0 + fi + echo "::error::Exit 0 with only $rows rows." + exit 1 + fi + echo "::error::The bare-runner run exited $code: neither a recognised refusal (3) nor real rows (0). 5 means content was not obtained (timeout or an unreadable page); 1 is a crash." + exit 1 + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: canary-datacentre + path: | + bare.json + bare.meta.json + bare_page1_debug.html + if-no-files-found: ignore diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1351131..42a4504 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: # check still imported the row dataclass by a name this repo renamed, # and it failed on this repo's FIRST push while the script it # duplicates passed — two sources of truth, one of them stale, which - # is exactly the defect CLAUDE.md §17 describes. + # is exactly how a check turns into dead code that looks load-bearing. # # So: one implementation, and `smoke_test.py` now asserts the workflow # does not grow another one. diff --git a/.gitignore b/.gitignore index ecc9bae..cd1618b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,11 +24,18 @@ live_results.zip *_debug.html *_debug.png *.html.page* +# `--dump-html PATH --pages N` writes PATH.pageN whatever PATH ends in, so +# `snap.json.page2` and `out.page3` are dumps too. +*.page[0-9]* +# Root-level HTML is a dump or a capture; the tracked landing page is listed +# by name below and is unaffected (ignoring a tracked file changes nothing). +/*.html # --- captures kept for parser work --------------------------------------- # The smoke suite's fixtures are committed (trimmed, inside smoke_test.py); # the multi-megabyte originals they were cut from are not. captures/ +live/ # --- the landing page and the proxy list --------------------------------- # Both live outside the repo on purpose: the landing page is published @@ -73,7 +80,7 @@ secrets* # recoverable only because that repo was private at the time. On 2026-09-17 # a sibling had an untracked `.env.bak` holding a 32-hex key in a PUBLIC # repository, one `git add -A` from publication. Craigslist was fixed then -# and the fix never reached anyone else, which is §16's rule unapplied: when +# and the fix never reached anyone else, which is the rule unapplied: when # you fix one, check the siblings. .env* !.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 9643fe7..4dc90fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,63 @@ does it leads the release notes rather than being discovered from a bill. ## [Unreleased] +> **Behaviour changes:** +> - **CSV cells that start with `=`, `+`, `-`, `@`, TAB, CR or LF now get a +> leading apostrophe** so a spreadsheet reads them as text. Only text +> cells, only in the CSV; numbers and the JSON output are unchanged, and +> the sidecar records how many cells were escaped as `csv_cells_escaped`. +> - **Counting flags reject values that skipped or crashed a run** with exit +> 2 naming the flag: `--pages` and `--concurrency` below 1, +> `--proxy-block-retries` below 0 (a negative value fetched nothing at +> all), `--delay` and `--retry-delay` below 0 (a negative delay crashed in +> `time.sleep`). In `scraper_api_client.py`, `--timeout` below 1 and +> `--retries`/`--retry-delay` below 0. +> - **A 404 now exits 4 in the Playwright and pyppeteer engines and in the +> Scraper API client**, as the 0.1.2 notes already said it did: the HTTP +> status now reaches the page classifier. Before, a removed job was +> waited on, re-fetched and reported as content not obtained (exit 5). +> Selenium exposes no HTTP status and still decides by the page body. +> - **`.env.example` ships `WELLFOUND_URL` commented out**, and an +> environment URL no longer collides with `--role`/`--location`/`--remote` +> or an explicit `--mode job`/`--mode jobs`: only a `--url` typed on the +> command line does. + +### Fixed + +- **Selenium and pyppeteer crashed when a page was refused and no proxy was + configured** — the normal outcome from a datacentre address. The block + retry touched `pool.current` with no pool and raised `AttributeError`; + pyppeteer also did so on a plain load timeout. Both now re-fetch through + the same path, as the Playwright engine does, and rotate only when a pool + of more than one exit exists. +- **Selenium and pyppeteer misread every `--mode job` page.** They + classified it as a listing, found no listing payload and reported a parse + error: zero rows, exit 5. Every classify call now passes the mode. +- **The canary skipped green for as long as it existed.** Without the + `WELLFOUND_PROXY` secret it now fails with an error naming the secret, + and a second job runs from the bare runner and requires the datacentre + refusal to be recognised as one (exit 3), not misread or crashed. The + exit-5 explanation in the canary named an old meaning. +- **The credential scan skipped everything when the clone lived under a + directory named `build`, `venv`, `.venv` and similar**, because the + exclusions were tested on the absolute path. They are now tested on the + path inside the repository, and a scan that reads 0 files fails. +- **`.gitignore`** now covers `--dump-html` page files whatever the base + name (`snap.json.page2`), a `live/` directory and root-level `.html` + dumps. +- **Comments and help text** no longer describe other sites' pages, a + `challenge` state this repo does not have, or a `--mode profile` it never + had, and no longer cite private working notes. + +### Tests + +- The offline suite now loads each engine with its driver library stubbed + and drives the real page loop, so the no-proxy block, the timeout, `--mode + job` and the 404 status are checked in every venv, not only where the + engine is installed. New checks also cover the environment URL with the + role flags, the counting-flag bounds, CSV formula escaping, the scan's + path handling and private citations in tracked files. + ## [0.1.2] — 2026-10-13 > **Behaviour change:** `--retries 0` is now a usage error (exit 2) instead diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54cafda..8461adf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,13 +107,13 @@ history needs a decision, not a red check on every push. Then the rest of the presentation, in the order that matters: -1. `python3 smoke_test.py` green, and the canary dispatched at least once — - including its SKIP branch, which is what runs when the `WELLFOUND_PROXY` - secret is absent. Unlike some siblings in this family, EVERY route on - this site is refused to a datacenter address, and a GitHub runner is one, - so the canary is gated in full rather than in half. Confirm the skip path - runs and goes green, not just the happy one — a check that is always red - teaches everyone to ignore checks. +1. `python3 smoke_test.py` green, and the canary dispatched at least once + with the `WELLFOUND_PROXY` secret set. EVERY route on this site is + refused to a datacenter address, and a GitHub runner is one, so the + `listing` job needs a residential exit; without the secret it FAILS with + an error naming it, because a canary that skipped has tested nothing. + The `datacentre` job runs without a secret and only confirms that the + runner's refusal is recognised as one (exit 3) rather than misread. 2. The repo description, homepage and topics set (see the family notes on what those should say). 3. Only then the row in the org profile README — and check it with an diff --git a/captcha_solver.py b/captcha_solver.py index c0d2273..31ffe51 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -59,7 +59,7 @@ (`TurnstileTaskProxyless`), enterprise reCAPTCHA (`RecaptchaV2EnterpriseTaskProxyless`) and ordinary v2/v3, and this module builds all of them. The only sentence this repo is entitled to is which task -types it implements (CLAUDE.md §19). +types it implements. PROVENANCE OF THE FIGURES BELOW ------------------------------- @@ -1175,16 +1175,14 @@ def turnstile_task_for(challenge: CaptchaChallenge) -> dict: # a JPEG of distorted text and a GET form). Roughly 190 lines of it, and none # of it is ported here, because this site has no such page. # -# What this site does instead is refuse a HEADLESS browser. Measured -# 2026-09-10 from five different addresses, four of them residential: Akamai -# answers with HTTP 403 and a 394-byte "Access Denied" page carrying a -# reference id — no form, no image, no widget, nothing for a solver to -# answer. And the trigger is the CLIENT rather than the address: the very -# same addresses were served HTTP 200 and the full catalogue by a browser -# with a real window. So the response to a block here is `--headful` or -# `--cdp-endpoint`, not a solve and not a better proxy, and -# product_parser.detect_page_state reports it as "blocked" rather than -# "challenge" precisely so no solve is attempted and nothing is charged. +# What this site does instead is refuse an ADDRESS. Measured 2026-09-17: +# Cloudflare answers a datacentre exit with HTTP 403 on every route, either +# as the branded "Security Check | Wellfound" page (no widget at all) or as +# the "Just a moment..." interstitial, while a residential exit was served +# 200 on every route this scraper reads. product_parser.detect_page_state +# reports both as "blocked"; the solve path builds a task only when a +# Turnstile sitekey was actually captured, so the widget-less refusal costs +# nothing, and a different exit is the move that clears it. # # The reCAPTCHA / hCaptcha / Turnstile machinery above IS kept, and that is a # deliberate asymmetry rather than an inconsistency. Detection stays broad @@ -1192,6 +1190,6 @@ def turnstile_task_for(challenge: CaptchaChallenge) -> dict: # what the address has been doing — a narrow list is how a challenge gets # reported as an empty page months later. A solver for a challenge this site # has never been observed to serve is dead code; a DETECTOR for one is cheap -# insurance: a detection that fires on a page whose lots have -# already rendered guards nothing, which is why the default is -# `when-blocked` and why it counts lot links before it spends. +# insurance: a detection that fires on a page whose job records +# have already rendered guards nothing, which is why the default is +# `when-blocked` and why it counts rendered matches before it spends. diff --git a/diff_runs.py b/diff_runs.py index 0fce0eb..9ddba3b 100644 --- a/diff_runs.py +++ b/diff_runs.py @@ -70,7 +70,7 @@ # What is worth watching on a job board, and nothing else. # # A price monitor's fields are absent because a job has no price — porting -# them would be dead code that looks load-bearing (CLAUDE.md §4). What +# them would be dead code that looks load-bearing. What # changes on a job listing is its PAY, its TERMS and the COMPANY behind it, # and those are what anyone diffs a job board for. # @@ -104,7 +104,7 @@ # has no expression for it. So diffing a landing run against a job run would # report each of these as a change on every row, and none of it would be # about the job. When the two rows disagree on `data_source`, they are -# reported as `source_changed` rather than as changes (§8: a difference that +# reported as `source_changed` rather than as changes (a difference that # comes with a provenance difference says something about our own two # snapshots, not about the site). DETAIL_ONLY_FIELDS = ( @@ -287,7 +287,7 @@ def _check_comparable(args) -> bool: # Wellfound offers no ordering control at all — there is no sort # parameter on any landing route and no `sort` column on the row — so # there is nothing here for such a guard to compare, and a guard with no - # input would pass for the wrong reason (CLAUDE.md §22). + # input would pass for the wrong reason. # # What DOES change which rows are in a file here is how many pages were # fetched, which the sidecar records and the note below reports. diff --git a/env_config.py b/env_config.py index d14e441..4d68baa 100644 --- a/env_config.py +++ b/env_config.py @@ -66,7 +66,7 @@ # A literal-only check reported both of those as CONFIGURED, so `cp # .env.example .env` followed by a run connected to cb.2captcha.com with the # string `{login}-zone-…` as its username and got a 401 — a confusing failure -# a long way from its cause, which is precisely what §3's rule exists to +# a long way from its cause, which is precisely what reading credentials from the environment exists to # prevent. Any `{…}` left in a value means the line was never filled in. _BRACED_PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") diff --git a/fingerprint_client.py b/fingerprint_client.py index e84ce1e..d9af906 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -331,7 +331,7 @@ def main() -> int: # loader. It used to read `os.environ` alone, which meant a key put in # `.env` exactly as the README and .env.example instruct worked for every # engine and failed HERE with "No API key" — a documented mechanism not - # applied on one path, which is the shape of half the defects §16 lists. + # applied on one path, which is the shape of a defect that recurs across copied code. # # `.env` has to be LOADED before it can be read: `env_value` looks at # os.environ, and `load_env` is what fills that from the file. Calling it diff --git a/output_writer.py b/output_writer.py index 2eab885..f937e63 100644 --- a/output_writer.py +++ b/output_writer.py @@ -32,8 +32,8 @@ commerce columns even where they are null, because on a shop a null price is a fact worth recording. Wellfound is a job board: it has no price, no currency on a product, no discount, no stock and no brand, and six columns -null on every row of every run of every mode is exactly what CLAUDE.md §9 -says must not exist. `bbb-scraper` and `quora-scraper` are the precedent. +null on every row of every run of every mode is a schema that must not +exist. `bbb-scraper` and `quora-scraper` are the precedent. What a job board has where a shop has a price is a RANGE, which is two columns and not one, plus an equity range that no shop has at all. Those @@ -44,7 +44,7 @@ What IS kept, byte-identical and in order, is the family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — so one column name works across the family and a consumer reading several of these repos reads the same first -five columns in the same order (§9). +five columns in the same order. `rating` is absent, and that one is a measurement ------------------------------------------------- @@ -83,13 +83,13 @@ class JobPosting: The family prefix — `source`, `scraped_at`, `url`, `sku`, `title` — is byte-identical and in this order across every repo in the family, so a - consumer reading several of them reads the same first five columns - (CLAUDE.md §9). Everything after it is Wellfound's. + consumer reading several of them reads the same first five columns. + Everything after it is Wellfound's. The commerce columns the shop repos carry (`price`, `currency`, `discount_pct`, `in_stock`, `brand`) are absent rather than null: a job board has none of them, and six columns null on every row of every run - is exactly what §9 says must not exist. `bbb-scraper` and + is a schema that must not exist. `bbb-scraper` and `quora-scraper` are the precedent. What a job board has instead of a price is a salary RANGE, which is two columns, not one. """ @@ -172,7 +172,7 @@ class JobPosting: # ---- provenance ----------------------------------------------------- # Which structured source this row was read out of: `apollo` for the # Next.js cache on a landing or feed page, `jsonld` for a detail page's - # JobPosting. CLAUDE.md §8 — provenance of a value goes in a column, and + # JobPosting. Provenance of a value goes in a column, and # `diff_runs.py` reports a difference that comes with a `data_source` # difference as `source_changed` rather than as a real change. data_source: Optional[str] = None @@ -255,6 +255,33 @@ def _csv_value(v: Any) -> Any: return v +# Leading characters a spreadsheet reads as the start of a formula (or, for +# TAB/CR/LF, that some importers strip to reveal one). Every string column +# here — title, company name, tagline, description, compensation — is text an +# employer typed, so a cell like `=HYPERLINK(...)` reaches the CSV verbatim +# unless it is neutralised. +CSV_FORMULA_LEADS = ("=", "+", "-", "@", "\t", "\r", "\n") + +# Prefixed to such a cell. Excel, Google Sheets and LibreOffice treat the +# apostrophe as "the rest of this cell is text" and do not show it; a +# consumer parsing the CSV with `csv` does see it, which is why the count goes +# in the sidecar rather than staying silent. +CSV_FORMULA_ESCAPE = "'" + + +def _csv_escape(v: Any) -> Any: + """Neutralise a formula-shaped cell. Returns (value, was_escaped). + + Only STRINGS are touched: escaping a number would turn `-5` into the text + `'-5` and break every sum over the column. CSV only — the JSON output + keeps the site's text exactly as served, and `csv_cells_escaped` in the + sidecar declares that the two files differ. + """ + if isinstance(v, str) and v.startswith(CSV_FORMULA_LEADS): + return CSV_FORMULA_ESCAPE + v, True + return v, False + + @contextlib.contextmanager def _atomic(path: str, newline: Optional[str] = None): """Write to a temporary file beside `path`, then rename over it. @@ -314,7 +341,7 @@ def write_json(rows: Sequence[Any], path: str) -> None: json.dump([asdict(r) for r in rows], f, ensure_ascii=False, indent=2) -def write_csv(rows: Sequence[Any], path: str, row_cls: Type = JobPosting) -> None: +def write_csv(rows: Sequence[Any], path: str, row_cls: Type = JobPosting) -> int: # An empty result still gets the header row. A zero-byte file makes a # consumer fail on read (no columns to parse) instead of reading a valid # table with zero rows — and "an empty result is still a well-formed @@ -322,12 +349,23 @@ def write_csv(rows: Sequence[Any], path: str, row_cls: Type = JobPosting) -> Non # # The header comes from `row_cls`, not from the first row, so an empty # run still writes the columns of the mode that produced it. + # + # Returns how many cells were formula-escaped. fieldnames = [f.name for f in fields(row_cls)] + escaped = 0 with _atomic(path, newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for r in rows: - writer.writerow({k: _csv_value(v) for k, v in asdict(r).items()}) + row = {} + for k, v in asdict(r).items(): + # Escaped AFTER the list join: a joined list is text too, and + # its first element can be formula-shaped. + value, was_escaped = _csv_escape(_csv_value(v)) + row[k] = value + escaped += was_escaped + writer.writerow(row) + return escaped # Exit code used when a run completes but produced nothing. Distinct from 1 @@ -496,14 +534,15 @@ def run_meta(status: str, stop_reason: str, pages_requested: int, } if extra: # Merged rather than nested under a key, so a consumer reads - # `shop_rating` at the top level beside `products`. Run fields win a + # `total_jobs` at the top level beside `products`. Run fields win a # name collision: a caller cannot accidentally overwrite `status`. meta.update({k: v for k, v in extra.items() if k not in meta}) return meta def save(rows: Sequence[Any], out_prefix: str, fmt: str, - allow_empty: bool = False, row_cls: Type = JobPosting) -> int: + allow_empty: bool = False, row_cls: Type = JobPosting, + stats: Optional[dict] = None) -> int: """Write JSON/CSV and return a process exit code. Returns 0 when rows were written, EXIT_NO_PRODUCTS when there were none. @@ -530,8 +569,17 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, write_json(rows, f"{out_prefix}.json") print(f"[+] Saved {len(rows)} jobs -> {out_prefix}.json") if fmt in ("csv", "both"): - write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) + escaped = write_csv(rows, f"{out_prefix}.csv", row_cls=row_cls) print(f"[+] Saved {len(rows)} jobs -> {out_prefix}.csv") + # Through `stats` rather than the return value, because the return + # IS the exit code. The caller puts it in the sidecar. + if stats is not None: + stats["csv_cells_escaped"] = escaped + if escaped: + print(f"[!] {escaped} CSV cell(s) began with a formula character " + f"and were prefixed with {CSV_FORMULA_ESCAPE!r} so a " + f"spreadsheet reads them as text. The JSON output is " + f"unchanged — see csv_cells_escaped in the sidecar.") return 0 if rows else EXIT_NO_PRODUCTS @@ -603,12 +651,18 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, # names has a hole for every name nobody added to it. complete = stop_reason in COMPLETE_STOP_REASONS and not pages_failed row_cls = ROW_CLASS_BY_MODE.get(mode, JobPosting) - rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls) + write_stats: dict = {} + rc = save(rows, out_prefix, fmt, allow_empty=allow_empty, row_cls=row_cls, + stats=write_stats) wrote_output = bool(rows) or allow_empty if wrote_output: status = "complete" if (rows and complete) else ( "partial" if rows else "failed") + # Merged UNDER the caller's `extra`: an engine's own facts win a name + # collision, and run fields still win over both (see run_meta). + if write_stats: + extra = {**write_stats, **(extra or {})} write_run_meta(out_prefix, run_meta( status=status, stop_reason=stop_reason, pages_requested=pages_requested, pages_completed=pages_completed, diff --git a/page_flow.py b/page_flow.py index 950aad9..e4896b8 100644 --- a/page_flow.py +++ b/page_flow.py @@ -2,7 +2,7 @@ page_flow.py ------------ The retry / solve / blocked decision, as DATA rather than as three copies of -an if-chain (CLAUDE.md §1). +an if-chain. Wellfound answers a request five ways, and four of them want a different response: @@ -21,7 +21,7 @@ Selenium's `execute_script` takes a function BODY with an explicit `return` while Playwright and pyppeteer take `() => expr`, so a shared snippet would quietly acquire one driver's dialect. The callbacks below are named for the -OPERATION instead, and each engine spells it in its own dialect (§1). +OPERATION instead, and each engine spells it in its own dialect. """ import argparse @@ -40,7 +40,7 @@ # How many job links mean "this response is a listing". # -# `> 1` on purpose, per CLAUDE.md §5: waiting for ONE match resolves on +# `> 1` on purpose: waiting for ONE match resolves on # something unrelated — a nav link, a footer — long before the grid is # really there. A landing page carried 32 to 45 job rows across the # captures and the thinnest full page held 32, so four is a floor that a @@ -50,8 +50,7 @@ # A listing page's readiness anchor for the browser engines. # # A job URL is `/jobs/{id}-{slug}` — a contract with search engines, not a -# build-generated class, which is §4's "anchor on a URL pattern, never a CSS -# class". Wellfound's own class names are Tailwind-style utility strings and +# build-generated class, so it is the anchor rather than a CSS class. Wellfound's own class names are Tailwind-style utility strings and # would churn on any deploy. # # Note this is a readiness signal only. The ROWS come out of `__NEXT_DATA__`, @@ -82,7 +81,7 @@ def content_timeout_ms(mode: str) -> int: # # Polled through `count(selector)` — a callback each engine implements with # its own `querySelectorAll` call — and NEVER by handing the browser a string -# to evaluate. CLAUDE.md §18: a site whose Content-Security-Policy omits +# to evaluate. A site whose Content-Security-Policy omits # `unsafe-eval` kills `wait_for_function` with an `EvalError` and takes the # run down with exit 1. Wellfound has not been measured for that, and the # cheap habit costs nothing on a site that would have allowed it. @@ -121,7 +120,7 @@ def classify(html: Optional[str], status: Optional[int] = None, against a callee that took `status` second, and both crashed on their first fetch — invisible to import, `--help`, `compileall` and 400+ green offline assertions, because none of those calls a function the way a live - run does (§17). `smoke_test.py` binds every engine's call against this + run does. `smoke_test.py` binds every engine's call against this signature for that reason. """ return detect_page_state(html or "", status, url, mode) @@ -164,7 +163,7 @@ def classify(html: Optional[str], status: Optional[int] = None, # A page built out of Wellfound's own assets whose payload we could not # read. NOT "zero jobs": a served page that links to jobs and parses to # nothing is OUR bug, and reporting it as an empty listing sends the - # reader to check the URL instead of the parser (CLAUDE.md §20). Worth + # reader to check the URL instead of the parser. Worth # one retry in case the document was caught mid-swap, and always worth a # dump. "parse_error": {"retry": True, "solve": False, "blocked": False, "parse": False, @@ -202,7 +201,7 @@ def content_obtained(state: str) -> bool: site having emptied out; the chain was `unknown` -> an outcome still marked ok -> stop_reason "completed" -> zero rows -> exit 4, with `finish_run`'s own exit-5 branch never reachable because nothing ever - told it the run had failed (CLAUDE.md §9, §25). + told it the run had failed. """ return STATE_POLICY.get(state, STATE_POLICY["unknown"])["obtained"] @@ -213,7 +212,7 @@ def stop_reason_for(load_failed: bool, blocked_by: Optional[str], One implementation because the engines derive this in seven places between them, and three copies of a triage drift silently — which is the - argument this module exists for (CLAUDE.md §1). `content_not_obtained_*` + argument this module exists for. `content_not_obtained_*` is deliberately NOT in `COMPLETE_STOP_REASONS`, so a zero-row run carrying it exits 5 rather than 4. """ @@ -247,7 +246,7 @@ def should_parse(state: str) -> bool: # True here, and CONSULTED rather than merely documented — the engines read # it, so setting it False really does stop the retry loop. (A sibling repo # carried this constant with a paragraph of justification and no reader, -# which is the same defect as dead code that looks load-bearing: §17.) +# which is the same defect as dead code that looks load-bearing.) # # True because on Wellfound a re-fetch genuinely can change the answer, and # the measurement says which kind of re-fetch. From one datacenter address @@ -260,7 +259,7 @@ def should_parse(state: str) -> bool: # CONSULTED rather than merely documented — the engines read it, so setting # it False really does stop the retry loop. (A sibling repo carried this # constant with a paragraph of justification and no reader, which is the -# same defect as dead code that looks load-bearing: CLAUDE.md §17.) +# same defect as dead code that looks load-bearing.) RETRY_ON_BLOCKED = True # How many times to re-fetch a blocked page when there is no proxy pool to @@ -315,10 +314,40 @@ def attempts(value: str) -> int: return n +def _number(value: str, kind, minimum, what: str): + try: + n = kind(value) + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not a number") + if n != n or n < minimum: # n != n: NaN + raise argparse.ArgumentTypeError(f"must be {what}, got {value}") + return n + + +def at_least_one(value: str) -> int: + """argparse type for a COUNT of pages or workers: 1 or more.""" + return _number(value, int, 1, "at least 1") + + +def non_negative(value: str) -> int: + """argparse type for a count of EXTRA tries (0 = none): 0 or more. + + A negative `--proxy-block-retries` made the block loop `range(0)`, so the + page was never fetched at all — the same silent no-op `--retries 0` was. + """ + return _number(value, int, 0, "0 or more") + + +def seconds(value: str) -> float: + """argparse type for a delay: 0 or more seconds. `time.sleep` raises on a + negative value, so `--delay -1` used to crash after page 1.""" + return _number(value, float, 0.0, "0 or more seconds") + + def pagination_is_addressable(url: str) -> bool: """Whether page N of this listing can be fetched without walking to it. - This is the question CLAUDE.md §18 says to ask PER URL rather than per + Whether pages are addressable is asked PER URL rather than per site, and on Wellfound the two page kinds give OPPOSITE answers: * a `/role/…` landing page takes `?page=N` and honours it. Measured @@ -362,6 +391,6 @@ def concurrency_limit(cdp_endpoint: Optional[str]) -> Optional[int]: The Scraping Browser API allows ONE live connection per profile, so N workers sharing a `pid` collide with `profile_locked`. Several `pid`s, - one run each, is the way to parallelise that path (§7). + one run each, is the way to parallelise that path. """ return 1 if cdp_endpoint else None diff --git a/playwright_scraper.py b/playwright_scraper.py index 4b300b3..52edc18 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -119,10 +119,10 @@ class PageOutcome: products: List = field(default_factory=list) blocked_by: Optional[str] = None load_failed: bool = False - # The page_flow state this page came back as ("content", "blocked", - # "challenge", "empty", "unknown"). Carried so the caller can tell an - # EMPTY page — a /p/ hub, a no-match query, or one page past the - # end of a listing — from a page that failed. Both produce zero rows and + # The page_flow state this page came back as ("content", "empty", + # "blocked", "parse_error", "not_found", "unknown"). Carried so the + # caller can tell an EMPTY page — a landing page that matched nothing — + # from a page that failed. Both produce zero rows and # they mean opposite things. state: Optional[str] = None # What Wellfound itself said the result set was: `totalJobCount`, @@ -224,7 +224,7 @@ def _driver(page): # `__NEXT_DATA__` in the first response. Every job record a landing page # holds — 37, 43, 45 and 32 on four captures — is in the document before # any scrolling could happen, so a scroll would be ceremony that looks - # load-bearing (CLAUDE.md §4). + # load-bearing. return { "count": lambda selector: len(page.query_selector_all(selector)), "sleep": page.wait_for_timeout, @@ -307,7 +307,7 @@ def _plan_page_urls(args, page_one_url: str, # The /jobs feed. `?page=2` on it returns the identical 46 ids as # page 1, so planning any page URL at all would manufacture # duplicates and report a complete multi-page run holding one page - # several times (CLAUDE.md §18). + # several times. if args.pages > 1: logger.info("This listing is served at ONE address — %s. " "`?page=2` on it returns the identical rows as page " @@ -394,8 +394,8 @@ def _launch_local(pw, args, pool): # navigation: Cloudflare calls `turnstile.render(container, params)` once # and keeps nothing, so `sitekey`, `action`, `cData` and `chlPageData` # exist only inside that call. A static read of a challenge page — any - # static read, however careful — cannot produce a solvable task - # (CLAUDE.md §19). It costs nothing on a page that never renders one. + # static read, however careful — cannot produce a solvable task. + # It costs nothing on a page that never renders one. context.add_init_script(TURNSTILE_INTERCEPT_JS) return browser, context, context.new_page() @@ -612,7 +612,7 @@ def handle_captcha_if_present(page, args) -> bool: about the variant and the parameters for one are rejected for the other. WHAT THIS SITE ACTUALLY PUTS IN FRONT OF A RUN, measured 2026-09-17, - because CLAUDE.md §19 is explicit that a sentence about what a solver + because a sentence about what a solver can do is the most expensive thing this family can get wrong: * Cloudflare's **managed challenge**, which every route answers with @@ -684,7 +684,7 @@ def handle_captcha_if_present(page, args) -> bool: # successful 107-row scrape. Ungated, `--solve-captcha always` would # have bought three solves for three pages that were never blocked. # - # CLAUDE.md §18 states the rule for the PARSER's marker set; this is + # A marker must be tested against good pages before it is trusted; this is # the same rule applied to the SOLVER's, which is a second marker set # in a second file and was not covered by it. if challenge is None and already_rendered <= MIN_CARD_MATCHES: @@ -700,7 +700,7 @@ def handle_captcha_if_present(page, args) -> bool: "publishes one in its own config — using it.") else: # Raising a task without a sitekey buys a rejected token and - # an ERROR_CAPTCHA_UNSOLVABLE bill (CLAUDE.md §19). Say so + # an ERROR_CAPTCHA_UNSOLVABLE bill. Say so # instead. logger.warning( "A Cloudflare Turnstile is on this page but no sitekey " @@ -795,7 +795,7 @@ def _solve_budget(args, spent: int): real Cloudflare challenge on every fetch: ONE page bought THREE Turnstile solves — two from the uncounted call across two block attempts, one from the counted one — and every token was refused. The cap read as enforced - and was not (CLAUDE.md §17: a policy constant nothing reads is the same + and was not (a policy constant nothing reads is the same defect as dead code). Both call sites now go through here. @@ -847,10 +847,17 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.info("Fetching page %d/%d: %s", page_num, args.pages, url) # Retry a navigation timeout rather than ending the run on it. One # network flap on page 12 of 50 should not break the loop. - load_failed, exit_failed = False, None + load_failed, exit_failed, status = False, None, None for attempt in range(1, args.retries + 1): try: - session.page.goto(url, wait_until="domcontentloaded", timeout=60000) + # The response is KEPT: its status is the classifier's + # primary signal (a 403 refusal, a 404 for a job that is + # gone). Discarding it made `not_found` unreachable, so a + # removed job was waited on, retried and reported as content + # never obtained. + response = session.page.goto(url, wait_until="domcontentloaded", + timeout=60000) + status = getattr(response, "status", None) load_failed = False break except (PWTimeout, PWError) as e: @@ -887,8 +894,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome solves_bought += 1 if handle_captcha_if_present(session.page, args): # A solve navigated the page. Give the destination a moment - # before judging what came back. + # before judging what came back, and judge it by its body: + # the first navigation's status no longer describes it. session.page.wait_for_timeout(1000) + status = None elif solves_bought: logger.info("Not solving again on page %d: %d purchase(s) already " "made for it and SOLVES_PER_PAGE is %d. A challenge " @@ -897,7 +906,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome page_flow.SOLVES_PER_PAGE) html = _snapshot(session.page, url) or "" - state = _classify(session.page, html, mode=args.mode) + state = _classify(session.page, html, status, mode=args.mode) # Wellfound server-renders its payload into `__NEXT_DATA__`, so a # listing is parseable in the FIRST response and there is nothing to @@ -926,21 +935,26 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.info("Still nothing after %.0fs (%d match(es) for %s).", wait_timeout / 1000, found, _ready_selector(args)) html = _snapshot(session.page, url) or html - state = _classify(session.page, html, mode=args.mode) - - # The paid path is reached only for state "challenge" — Cloudflare's - # managed challenge, which IS a test and can be solved once - # `TURNSTILE_INTERCEPT_JS` has captured its parameters. It is bounded + # No status here: the document may have changed during the wait + # (a challenge that clears itself navigates), so the navigation's + # status no longer describes it and the body decides. + state = _classify(session.page, html, None, mode=args.mode) + + # The paid path is reached only for a state the policy marks solvable + # ("blocked" here) — Cloudflare's managed challenge IS a test and can + # be solved once `TURNSTILE_INTERCEPT_JS` has captured its parameters, + # while the widget-less branded refusal has nothing to solve. It is bounded # by SOLVES_PER_PAGE so a rotation loop cannot become a bill, and a # sitekey-less detection refuses to build a task at all rather than - # paying for one the API will reject (CLAUDE.md §19). + # paying for one the API will reject. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): solves_bought += 1 if handle_captcha_if_present(session.page, args): session.page.wait_for_timeout(1000) html = _snapshot(session.page, url) or html - state = _classify(session.page, html, mode=args.mode) + # The solve reloaded the page; the original status is stale. + state = _classify(session.page, html, None, mode=args.mode) # The VERIFIED outcome, and the only one worth reporting: a # "ready" task result is not evidence the token works. This # line is what says whether the money bought anything. @@ -954,7 +968,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if not page_flow.should_retry(state): # "content" and "empty" are both final answers. An empty page is - # a CORRECT one — a hub category has no grid, and one page past + # a CORRECT one — a landing page that matched nothing, and one page past # the end of a listing has no products — so retrying it would # spend the user's budget re-confirming the same right answer, # and rotating the exit would blame an address for the URL it was @@ -990,7 +1004,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome time.sleep(pause) if load_failed: - # NAME the proxy when it was the proxy. CLAUDE.md §8: a dead exit and + # NAME the proxy when it was the proxy. A dead exit and # a timeout want opposite responses — another try at the same exit # versus a different exit — so a message that cannot tell them apart # leaves the reader guessing which they got. @@ -1021,11 +1035,12 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # What a caller needs here is WHICH refusal this is, because the two # want different answers and only one of them is solvable. # - # "You have been blocked | Better Business Bureau®" — a refusal. - # No widget, no sitekey, nothing to solve. A different exit is - # the only move. - # "Just a moment..." with `cf_chl_opt` — a test. - # That classifies as "challenge", not here, and IS solvable. + # "Security Check | Wellfound" — a refusal. No widget, + # no sitekey, nothing to solve. A different exit is the only + # move. + # "Just a moment..." with `_cf_chl_opt` — a test. It is the same + # "blocked" state, but it carries a widget, so the solve path + # above may have tried it once. # # Both refusals are HTTP 403 and the branded one wears Wellfound's # own name four times, so a reader who only sees "403" cannot tell @@ -1059,7 +1074,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Wellfound embeds the whole result set in the first response, so there # is nothing to wait for and nothing to scroll into view. Porting the # sibling repos' scroll loop here would be dead code that looks - # load-bearing (CLAUDE.md §4). + # load-bearing. # Dumping on success, not only on failure: a run can return the right # NUMBER of rows with a field silently unpopulated, and then the only way @@ -1080,10 +1095,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # is the "detected is not blocking" rule the captcha default follows, # applied to the blocking decision instead of the spending one. But # `state != "content"` is still too wide: an EMPTY page is a correct - # answer, and a live run of a /p/ hub reported exit 3 on a 191 KB - # page the site had plainly served, because the hub's own performance - # script names `akamaihd.net` and "akamai" was in the marker list. Both - # halves were wrong; the marker is gone (see + # answer, and a sibling scraper reported exit 3 on a 191 KB page its + # site had plainly served, because a performance script named a vendor + # host that was in the marker list. Both halves were wrong; that marker + # is gone (see # product_parser.BOT_CHALLENGE_MARKERS) and this now only refines the # REASON for a page the policy had already given up on. vendor = (detect_bot_challenge(html, url=session.page.url) @@ -1111,7 +1126,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # something its twins would not. # Dump it. A state that holds no rows is a state somebody will have # to diagnose, and the only way to tell a parsing bug from a page - # that never arrived is the exact bytes (CLAUDE.md §9). Until the + # that never arrived is the exact bytes. Until the # 2026-10-13 audit this path wrote nothing at all: the auditor's run # failed and left no artefact, which is half of why the cause was # misread. The blocked path has always dumped; this one did not. @@ -1312,7 +1327,7 @@ def scrape(args) -> int: outcomes: List[PageOutcome] = [] seen_keys = set() blocked = False - # All three modes are one row per business-at-a-location, so `sku` is the + # All three modes are one row per job posting, so `sku` (the job id) is the # key for all of them. dedupe_key = "sku" # Why the loop ended. "completed" means every requested page was fetched; @@ -1621,7 +1636,7 @@ def parse_args(): "category tree, so this is a name for the QUERY " "rather than a site concept; it exists so two runs of " "different roles are not mistaken for comparable.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.at_least_one, default=1, help="Number of listing pages to fetch. Applies to --mode " "role only. A run PLANS against the `pageCount` the " "site states on page 1 and never asks past it — " @@ -1630,8 +1645,8 @@ def parse_args(): "a run that trusted the request would collect twice. " "A request for more is honoured up to that cap and " "the sidecar records both numbers.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.seconds, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, metavar="N", help="Fetch pages through N parallel workers (default 1 — " "unchanged sequential behaviour). Each worker runs its " "own browser and holds its own proxy exit, so N>1 " @@ -1641,9 +1656,9 @@ def parse_args(): help="Attempts per page load before giving up (default 3). " "The pause between attempts doubles each time. A page " "that comes back EMPTY is not retried — see " - "page_flow.STATE_POLICY — because an empty hub " - "category is a correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + "page_flow.STATE_POLICY — because an empty " + "landing page is a correct answer, not a fault.") + p.add_argument("--retry-delay", type=page_flow.seconds, default=2.0, help="Seconds before the first page-load retry, doubling " "thereafter (default 2.0)") p.add_argument("--format", choices=["json", "csv", "both"], default="both") @@ -1667,7 +1682,7 @@ def parse_args(): p.add_argument("--proxy-shuffle", action="store_true", help="Shuffle the pool at startup, so concurrent runs do not " "all begin on the first exit in the file.") - p.add_argument("--proxy-block-retries", type=int, default=2, + p.add_argument("--proxy-block-retries", type=page_flow.non_negative, default=2, help="When a page comes back refused (HTTP 403) or behind " "a captcha, retry it from this many OTHER exits before " "giving up (default 2). Needs a pool of more than one; " @@ -1736,15 +1751,28 @@ def parse_args(): p.add_argument("--headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") args = p.parse_args() + # What was TYPED, captured before the environment fills anything in. + typed_url = args.url # Fill --twocaptcha-key / --cdp-endpoint / --proxy / --url from the # environment or .env when the flag was not given. An explicit flag wins. env_config.apply(args) + # A WELLFOUND_URL from the environment or .env is a default target, not + # a query the user typed, so it yields to the role flags and to an + # explicit --mode job/jobs. Without this, a copied .env.example made + # `--role X` and `--mode jobs` exit 2 as if --url had been typed with them. + if not typed_url and args.url and ( + args.role or args.location or args.remote + or (args.mode in ("job", "jobs") + and mode_for_url(args.url) not in (None, args.mode))): + logger.info("Not using WELLFOUND_URL from the environment: the " + "command line selects the query.") + args.url = None # --url and the role flags are two ways to say the same thing, and only # one of them may win. Refused rather than merged: a --location that # disagreed with the one already in a /role/l/… path would silently # scrape a different listing than the address names, which is the hazard - # CLAUDE.md §10 bans a --country flag for, in the form it takes here. + # a flag that overrides part of a URL always carries. if args.url and (args.role or args.location or args.remote): conflicting = [name for name, value in (("--role", args.role), ("--location", args.location), @@ -1780,13 +1808,13 @@ def parse_args(): # Refused rather than attempted. This parser reads Wellfound's own # Apollo cache and its JobPosting block; pointing it at another site # would not fail loudly, it would return zero rows and look like an - # empty result (§8). The REASON is given, because a wrong reason + # empty result. The REASON is given, because a wrong reason # about a host that plainly is one sends the reader hunting a typo. p.error(f"{args.url!r} {why}.") # The URL knows which mode it belongs to, so a disagreement is caught - # here rather than producing zero rows from the wrong reader. CLAUDE.md - # §8: a parser pointed at the wrong page kind fails silently, and silence + # here rather than producing zero rows from the wrong reader. + # A parser pointed at the wrong page kind fails silently, and silence # is the failure this check exists to convert into an error message. url_mode = mode_for_url(args.url) if url_mode and url_mode != args.mode: diff --git a/product_parser.py b/product_parser.py index 2e90592..d11ebd6 100644 --- a/product_parser.py +++ b/product_parser.py @@ -6,8 +6,8 @@ Where the data actually is -------------------------- Counted on 2026-09-17 over the captures in `captures/wellfound`, because -CLAUDE.md §15 says to count the JSON-LD blocks before writing a line of -parser and the answer decides the whole design: +The JSON-LD blocks were counted before writing a line of parser, and the +answer decides the whole design: /role/{role} 0 JSON-LD blocks /role/r/{role} 0 JSON-LD blocks @@ -27,8 +27,8 @@ server-rendered by a different stack — and publishes one schema.org `JobPosting` instead. -This is CLAUDE.md §20's "a DETAIL page may publish a different JSON-LD type -than the listing", in its sharpest form: the detail page publishes a +A DETAIL page may publish a different JSON-LD type than the listing, and +here that happens in its sharpest form: the detail page publishes a different *mechanism*, not merely a different type. Porting the listing parser to a detail page returns zero rows in silence, so `parse_detail()` is a separate function and `smoke_test.py` pins that failure directly. @@ -39,7 +39,7 @@ strict subset of it, and the class names are build hashes. A DOM path would be a worse copy of the same data. What replaces it is a shape check: if `__NEXT_DATA__` is present and parses but yields no job records, that is -`parse_error` rather than "empty category" (§20). +`parse_error` rather than "empty category". Three routes, one query ----------------------- @@ -67,7 +67,7 @@ answers HTTP 200.** `?page=48` against a 47-page listing returned the same 37 job ids as page 1, byte for byte. A run that walked past the end would re-collect page 1 for as long as it was asked to. The "no new sku" -terminator (§7 layer 3) does catch it, but there is a stronger and +terminator does catch it, but there is a stronger and unambiguous signal available and `echoed_page()` reads it: the Apollo cache key embeds the arguments the SERVER used, so a response to a `?page=48` request is keyed `seoLandingPageJobSearchResults({"page":1,...})` and says @@ -75,8 +75,7 @@ Trap 2 — **`/jobs` is not addressable at all.** `?page=2` on it does not fail and does not empty; it returns the identical 46 job ids as page 1. -This is CLAUDE.md §18's "one page kind may have no per-page addresses", -so `pagination_is_addressable()` answers False for it, the engines refuse +One page kind may have no per-page addresses, so `pagination_is_addressable()` answers False for it, the engines refuse `--concurrency > 1` with that reason, and a `--pages 5` run of that mode stops after one page with `stop_reason=single_page_listing` rather than reporting a complete 5-page run holding one page five times. @@ -108,8 +107,8 @@ as 30 to 80. * currency comes from the symbol, and only these four appear: `$` 576, `£` 16, `₹` 8, `€` 2. `$` is read as USD because that is what it means - on a US job board and nothing better is on the page — CLAUDE.md §4's - rung 4, a guess, and `salary_currency` is null rather than defaulted + on a US job board and nothing better is on the page — a + guess, and `salary_currency` is null rather than defaulted when no symbol is present at all. * `'No equity'` is a STATEMENT, not a missing value: it means the offer has none. It is recorded as `has_equity=False`, which is different from @@ -118,7 +117,7 @@ The listing string carries no period. `$140k` on a job board reads as a year, but the page does not say so, so `salary_period` stays null on a listing row and is filled only from the detail page's JSON-LD, which states -`unitText: "YEAR"`. §8: never present a guess as a fact. +`unitText: "YEAR"`. Never present a guess as a fact. Where both views exist they agree — see `smoke_test.py`, which pins job 4697947 at 100000–180000 USD from the detail page against `'$100k – $180k @@ -166,13 +165,13 @@ # spelling of the same host would put two spellings into the output for one # job. The refusal says the host redirects and asks for the wellfound.com # form — it does NOT claim the address is dead, because it is not -# (CLAUDE.md §5: a refusal with a false reason sends the reader hunting for +# (a refusal with a false reason sends the reader hunting for # a typo). LEGACY_HOSTS = ("angel.co", "www.angel.co", "angellist.com", "www.angellist.com") # The one address `--mode jobs` reads. Named HERE rather than written out in # each engine: three copies of a URL literal is three places for it to drift, -# and CLAUDE.md §1 puts site knowledge in this module and a handful of named +# and site knowledge belongs in this module and a handful of named # constants in the engines — not a bare URL in all three. FEED_URL = BASE_URL + "/jobs" @@ -245,7 +244,7 @@ def _epoch_to_iso(value: Any) -> Optional[str]: Guarded against a value out of range rather than trusted: a bad timestamp raises inside `utcfromtimestamp`, and one malformed row must - not take a page of good ones down with it (§8, fail loudly but locally). + not take a page of good ones down with it (fail loudly but locally). """ if isinstance(value, bool) or not isinstance(value, (int, float)): return None @@ -297,7 +296,7 @@ def _text_list(values: Any) -> Optional[List[str]]: _COMP_SEPARATORS = ("•", "|", "·") # Only these four were measured. An allowlist rather than a general symbol -# class, for the same reason CLAUDE.md §4 wants an ISO-code allowlist: a +# class, for the same reason a currency code needs an allowlist: a # stray glyph beside a number must not mint a currency. _CURRENCY_BY_SYMBOL = {"$": "USD", "£": "GBP", "€": "EUR", "₹": "INR"} @@ -431,7 +430,7 @@ def route_of(url: str) -> Optional[str]: def is_supported_url(url: str) -> Tuple[bool, str]: """(ok, reason). The reason is shown to the user, so it says WHY. - CLAUDE.md §5: refusing a host with a false reason sends the reader + Refusing a host with a false reason sends the reader hunting for a typo. `angel.co` really is Wellfound — it redirects here — so the refusal says that rather than "not a Wellfound URL". """ @@ -488,7 +487,7 @@ def _query_pairs(query: str) -> List[Tuple[str, str]]: def page_url(url: str, page: int) -> str: """Replace `?page=N`, preserving every other query parameter. - CLAUDE.md §5: replace rather than duplicate, and keep what was there — + Replace rather than duplicate, and keep what was there — a landing URL may carry campaign parameters the user pasted and losing them changes what the site answers. """ @@ -595,7 +594,7 @@ def apollo_state(payload: Any) -> Optional[Dict[str, Any]]: a wrapper whose only key is `data`. Reading `apolloState` itself gives a one-entry dict whose values have no `__typename`, which looks like an empty page rather than like a wrong path — the exact silent-zero failure - §4 lists. Accepts either level so a caller that already unwrapped one + to avoid. Accepts either level so a caller that already unwrapped one gets the same answer. """ if isinstance(payload, str): @@ -642,7 +641,7 @@ def echoed_page(state: Optional[Dict[str, Any]]) -> Optional[int]: `?page=48` against a 47-page listing comes back keyed `{"page":1,…}`. Comparing that against the page that was requested turns "the site silently re-served page 1" into an unambiguous end-of-listing signal, - which CLAUDE.md §17 prefers over the weaker no-new-sku heuristic. + which is stronger than the weaker no-new-sku heuristic. """ if not isinstance(state, dict): return None @@ -701,7 +700,7 @@ def _company_of(state: Dict[str, Any], job_key: str, The upward case is why this takes the job's cache KEY: that key is what the company's list holds, and matching on it is exact. Scanning for a company whose list contains this job is the same "stop at the ancestor - that covers exactly one product" discipline §4 describes for tiles, + that covers exactly one product" discipline used for product tiles, done against ids instead of DOM depth. """ direct = _deref(state, job.get("startup")) @@ -783,7 +782,7 @@ def parse_listing(source: Any, *, url: str = "", page: Optional[int] = None, server serialised them and therefore the order the page paints. That makes `position` meaningful; `page` is threaded in from the caller rather than inferred, because a parser cannot know which page it was - handed and CLAUDE.md §18 names the bug where every row claimed page 1. + handed, and guessing produced rows that all claimed page 1. """ state = apollo_state(source) if not _looks_like_state(source) else source if not isinstance(state, dict): @@ -805,7 +804,7 @@ def _looks_like_state(source: Any) -> bool: def listing_meta(state: Optional[Dict[str, Any]]) -> Dict[str, Any]: """The site's own pagination arithmetic, for the run sidecar. - CLAUDE.md §21: a sidecar that says only "complete" is lying by omission + A sidecar that says only "complete" is lying by omission where the site caps what it will serve. Wellfound does not cap the way bbb-scraper's site does — 47 pages really is all 923 companies — but it counts COMPANIES per page while a consumer counts jobs, so `total_jobs` beside @@ -1011,7 +1010,7 @@ def parse_detail(html: str, url: str = "", *, # before it was written down — five served captures (three landing shapes, # the /jobs feed, a job detail page) against two refusals (the branded 403 # a plain HTTP client gets, and the /company/ page a real Chromium on a -# residential exit gets). CLAUDE.md §18: a marker that matches a good page +# residential exit gets). A marker that matches a good page # is worse than no marker. # # The counts, 2026-09-17, as served-max vs refused-min: @@ -1025,7 +1024,7 @@ def parse_detail(html: str, url: str = "", *, # cdn-cgi/challenge-platform 1 vs 1 REJECTED — both # # The rejected three are worth naming because two of them are what this -# family's own notes recommend. CLAUDE.md §19 promotes +# family's own notes recommend. Those notes promote # `challenges.cloudflare.com` as "the one that works" on the strength of # foodpanda-scraper and bbb-scraper measurements; on Wellfound it is exactly # BACKWARDS. @@ -1058,9 +1057,9 @@ def parse_detail(html: str, url: str = "", *, ("cloudflare", "Cloudflare Ray ID"), ) -# The hosts a page the site actually served is BUILT out of. CLAUDE.md §8's +# The hosts a page the site actually served is BUILT out of. The # positive-asset test, which is the only signal that answers correctly for -# Chromium's own network-error page (§18) — that page carries the site's +# Chromium's own network-error page — that page carries the site's # hostname in its and nothing else of the site's. SERVED_ASSET_HOSTS = ("photos.wellfound.com", "/_next/static") @@ -1077,7 +1076,7 @@ def parse_detail(html: str, url: str = "", *, def site_turnstile_sitekey(html: str) -> Optional[str]: """The Turnstile sitekey the site publishes in its own page config. - This is the answer to CLAUDE.md §18's real question — not "did we meet + This answers the real question — not "did we meet a captcha" but "is one configured, and would we recognise it". One is: the NEXT.JS routes carry this key — the landings and the /jobs feed — and the site renders a widget with it into `#turnstile_widget` when its @@ -1088,7 +1087,7 @@ def site_turnstile_sitekey(html: str) -> Optional[str]: script but publish no key in their config. That widget IS solvable from a static read, because the sitekey is published — unlike Cloudflare's own interstitial, which publishes none - and needs the `turnstile.render` interception hook §19 describes. + and needs the `turnstile.render` interception hook. """ m = _SITEKEY_RE.search(html or "") return m.group("key") if m else None @@ -1098,7 +1097,7 @@ def detect_bot_challenge(html: str, url: str = "") -> Optional[str]: """The vendor refusing this page, or None. Only ever called for a page the policy has already decided is not - content — CLAUDE.md §18's gate, so this can never fire on a correct + content — that gate is why this can never fire on a correct "no results" answer and turn it into exit 3. """ if not html: @@ -1118,7 +1117,7 @@ def references_own_assets(html: str) -> int: # The site's own copy for a landing page that matched nothing. An # UNAMBIGUOUS positive signal, so it is checked BEFORE the asset-count -# heuristic — CLAUDE.md §17's classification-order trap, where a minimal +# heuristic — the classification-order trap, where a minimal # real page with few asset references came back as "blocked". NO_RESULTS_MARKERS = ( "No jobs found", @@ -1135,7 +1134,7 @@ def detect_page_state(html: Optional[str], status: Optional[int] = None, Ordered by how much each signal PROVES, not by how cheap it is. 1. HTTP 403 is unambiguous on this site and is the primary signal, the - way §8 takes MediaMarkt's status as primary: every refusal measured + way a status is primary on any site that states one: every refusal measured came with 403 and every served page with 200. It is checked before any markup, because the branded refusal is titled "Security Check | Wellfound" and carries the site's name four times — a text or title @@ -1172,7 +1171,7 @@ def detect_page_state(html: Optional[str], status: Optional[int] = None, return "empty" # A Next.js page the site served, whose cache holds no job nodes # and which does not say it found nothing, is OUR failure to read - # it — never "0 jobs" (CLAUDE.md §20). + # it — never "0 jobs". return "parse_error" lowered = text.lower() diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index e8e3a01..3171860 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -647,8 +647,8 @@ def _solve_budget(args, spent: int): Measured 2026-09-17 from a datacenter address, which meets a real Cloudflare challenge on every fetch: ONE page bought THREE Turnstile - solves, and every token was refused. The cap read as enforced and was not - (CLAUDE.md §17). Both call sites now go through here. + solves, and every token was refused. The cap read as enforced and was not. + Both call sites now go through here. """ return spent < page_flow.SOLVES_PER_PAGE @@ -688,12 +688,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # `exit_failed` holds Chromium's own name for a proxy fault, or None. # Named and shaped exactly as in the other two engines: the give-up # branch below reports it, and three engines that structure this - # differently drift (CLAUDE.md §6). - load_failed, exit_failed = False, None + # differently drift. + load_failed, exit_failed, status = False, None, None for attempt in range(1, args.retries + 1): try: - bridge.run(page.goto(url, {"waitUntil": "domcontentloaded", - "timeout": 60000})) + # The response is kept: its status is the classifier's primary + # signal (403 refusal, 404 for a job that is gone). + response = bridge.run(page.goto( + url, {"waitUntil": "domcontentloaded", "timeout": 60000})) + status = getattr(response, "status", None) load_failed = False break except Exception as e: # noqa: BLE001 — pyppeteer raises many types @@ -716,8 +719,15 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome args.retries, text[:120], pause) time.sleep(pause) - if load_failed and block_attempt < block_retries: - pool.advance("unusable exit or repeated load failure") + # Only a dead EXIT is worth a rotation, and only when there is another + # exit to rotate to — the same gate as the Playwright engine. This used + # to rotate on any load failure and called `pool.advance` with no pool + # at all, raising AttributeError on a plain timeout without --proxy. + if exit_failed and has_pool and block_attempt < block_retries: + logger.warning("Exit %s is unusable (%s) — rotating to another " + "one (%d/%d).", mask(pool.current), exit_failed, + block_attempt + 1, block_retries) + pool.advance(f"unusable exit: {exit_failed}") session.relaunch() bridge, page = session.bridge, session.page d = _driver(session) @@ -731,13 +741,16 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome solves_bought += 1 if handle_captcha_if_present(session, args): time.sleep(1) + status = None # the solve reloaded; that status is stale elif solves_bought: logger.info("Not solving again on page %d: %d purchase(s) " "already made for it and SOLVES_PER_PAGE is %d.", page_num, solves_bought, page_flow.SOLVES_PER_PAGE) html = _snapshot(session, url) or "" - state = page_flow.classify(html, None, page.url) + # `mode` is passed because a job page carries no listing payload: + # classified as a listing, every one came out `parse_error`. + state = page_flow.classify(html, status, page.url, mode=args.mode) # Wellfound server-renders its payload, so a listing is parseable # in the FIRST response. The wait below is only for the state that @@ -759,24 +772,22 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.info("Still nothing after %.0fs (%d match(es) for %s).", wait_ms / 1000.0, found, sel) html = _snapshot(session, url) or html - state = page_flow.classify(html, None, page.url) - - # The paid path is reached only for state "challenge" — Cloudflare's - # Managed Challenge, which IS a test. It is NOT reached for - # "blocked": that page carries no widget, so a solve there would be a - # charge for nothing. Mirrors the other two engines. - # - # The paid path is reached only for state "challenge", which no - # capture of this site has ever produced. Wired up because a bot - # manager can be switched on between deploys, and bounded by - # SOLVES_PER_PAGE so a speculative path cannot become a bill. + # No status: the document may have changed during the wait. + state = page_flow.classify(html, None, page.url, mode=args.mode) + + # The paid path is reached only for a state the policy marks solvable + # ("blocked" here). Cloudflare's managed challenge IS a test; the + # branded refusal carries no widget and no sitekey, and the solver + # refuses to build a task without one, so it costs nothing there. + # Bounded by SOLVES_PER_PAGE. Mirrors the other two engines. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): solves_bought += 1 if handle_captcha_if_present(session, args): time.sleep(1) html = _content(session) or html - state = page_flow.classify(html, url=page.url) + state = page_flow.classify(html, None, page.url, + mode=args.mode) if state == "content": logger.info("The solve was accepted — page %d is content " "now.", page_num) @@ -787,7 +798,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if not page_flow.should_retry(state): # "content" and "empty" are both final answers. An empty page is - # a CORRECT one — a hub category has no grid — so retrying it + # a CORRECT one — a landing page that matched nothing — so retrying it # would re-confirm the same right answer, and rotating the exit # would blame an address for the URL it was given. break @@ -796,16 +807,29 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # so a different exit is the only thing that plausibly changes the # outcome. if block_attempt < block_retries: - logger.warning("Page %d came back as %s from %s — retrying from " - "another exit (%d/%d).", page_num, state, - mask(pool.current), block_attempt + 1, block_retries) - pool.advance(f"{state} on page {page_num}") - session.relaunch() - bridge, page = session.bridge, session.page - d = _driver(session) + if has_pool: + logger.warning("Page %d came back as %s from %s — retrying " + "from another exit (%d/%d).", page_num, state, + mask(pool.current), block_attempt + 1, + block_retries) + pool.advance(f"{state} on page {page_num}") + session.relaunch() + bridge, page = session.bridge, session.page + d = _driver(session) + else: + # No pool, so no other exit: re-fetch through the same access + # path, as the Playwright engine does. This branch used to + # touch `pool.current` unconditionally and raised + # AttributeError on the ordinary no-proxy block. + pause = args.retry_delay * (block_attempt + 1) + logger.warning("Page %d came back as %s — re-fetching through " + "the same access path in %.1fs (%d/%d).", + page_num, state, pause, block_attempt + 1, + block_retries) + time.sleep(pause) if load_failed: - # NAME the proxy when it was the proxy. CLAUDE.md §8: a dead exit and + # NAME the proxy when it was the proxy. A dead exit and # a timeout want opposite responses — another try at the same exit # versus a different exit — so a message that cannot tell them apart # leaves the reader guessing which they got. @@ -860,11 +884,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.info("Saved the snapshot the parser sees to %s (%d bytes).", dump_path, len(html)) - # Only for a state page_flow already counts as BLOCKED. An EMPTY - # page is a correct answer, and a live run of a /p/<slug> hub - # reported exit 3 on a page the site had plainly served because the - # hub's own performance script names `akamaihd.net`. Mirrors - # playwright_scraper exactly. + # Only for a state page_flow already counts as BLOCKED: an EMPTY page + # is a correct answer and must never become exit 3 because a marker + # matched it. Mirrors playwright_scraper exactly. vendor = (detect_bot_challenge(html, url=page.url) if page_flow.counts_as_blocked(state) else None) if vendor: @@ -885,7 +907,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if not page_flow.should_parse(state): # Dump it. A state that holds no rows is a state somebody will have # to diagnose, and the only way to tell a parsing bug from a page - # that never arrived is the exact bytes (CLAUDE.md §9). Until the + # that never arrived is the exact bytes. Until the # 2026-10-13 audit this path wrote nothing at all: the auditor's run # failed and left no artefact, which is half of why the cause was # misread. The blocked path has always dumped; this one did not. @@ -984,13 +1006,10 @@ def scrape(args) -> int: outcomes: List[PageOutcome] = [] seen_keys = set() blocked = False - # All three modes are one row per business-at-a-location. + # All three modes are one row per job posting, keyed by its id. dedupe_key = "sku" - # Only --mode product is single-page. A SHOP FRONT paginates exactly like - # a category listing — ?page=N, the same tiles — and treating it as - # single-page made `--mode shop --pages 2` fetch one page and report - # "complete", which is the silent-success failure this family exists to - # avoid. Found on the first live shop run. + # --mode job is one page by construction and --mode jobs is a feed + # served at one address; only --mode role paginates. stop_reason = ("single_page_mode" if args.mode == "job" else "single_page_listing" if args.mode == "jobs" else "completed") @@ -1166,20 +1185,20 @@ def parse_args(): help="Label to tag the run with in the sidecar. Defaults " "to what the URL selects. Wellfound has no category " "tree, so this names the QUERY, not a site concept.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.at_least_one, default=1, help="Listing pages to fetch (--mode role only). A run " "plans against the `pageCount` the site states on " "page 1 and never asks past it: page 48 of a 47-page " "listing answers HTTP 200 with page 1 AGAIN.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.seconds, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, metavar="N", help="Accepted for flag parity and IGNORED here: parallel " "page fetching lives in playwright_scraper.py.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3). " "A page that comes back EMPTY is not retried: an empty " - "hub category is a correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + "landing page is a correct answer, not a fault.") + p.add_argument("--retry-delay", type=page_flow.seconds, default=2.0, help="Seconds before the first retry, doubling thereafter") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="wellfound_jobs", help="Output file prefix") @@ -1192,7 +1211,7 @@ def parse_args(): "Wins over --proxy.") p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=page_flow.non_negative, default=2) p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") @@ -1248,7 +1267,22 @@ def parse_args(): p.add_argument("--headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") args = p.parse_args() + # What was TYPED, captured before the environment fills anything in. + typed_url = args.url + # Fill --twocaptcha-key / --cdp-endpoint / --proxy / --url from the + # environment or .env when the flag was not given. An explicit flag wins. env_config.apply(args) + # A WELLFOUND_URL from the environment or .env is a default target, not + # a query the user typed, so it yields to the role flags and to an + # explicit --mode job/jobs. Without this, a copied .env.example made + # `--role X` and `--mode jobs` exit 2 as if --url had been typed with them. + if not typed_url and args.url and ( + args.role or args.location or args.remote + or (args.mode in ("job", "jobs") + and mode_for_url(args.url) not in (None, args.mode))): + logger.info("Not using WELLFOUND_URL from the environment: the " + "command line selects the query.") + args.url = None # Spelled identically to the other two engines. if args.url and (args.role or args.location or args.remote): diff --git a/scraper_api_client.py b/scraper_api_client.py index bfd3519..ee43028 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -28,7 +28,7 @@ # TWOCAPTCHA_KEY and the CDP endpoint are read from .env as # $WELLFOUND_CDP_ENDPOINT, so neither needs to be typed — a secret in - # argv is readable by anything that can run `ps` (CLAUDE.md §3). + # argv is readable by anything that can run `ps`. """ import argparse @@ -48,6 +48,7 @@ parse_products, parse_detail) from output_writer import finish_run, SOURCE_DEFAULT import env_config +import page_flow logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("scraper_api_client") @@ -244,7 +245,23 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: # Same policy as the browser engines: the status decides the blocked # case, because this site's refusal has no marker to detect. - state = detect_page_state(html, status=upstream_status, url=args.url) + state = detect_page_state(html, status=upstream_status, url=args.url, + mode=args.mode) + if state == "not_found": + # A 404 is the site's own definitive answer — the job is gone — so it + # is a genuine empty result (exit 4), not a parse failure, exactly as + # in the browser engines. + logger.warning("Wellfound answered HTTP 404 for %s: nothing there.", + args.url) + return finish_run([], args.out, args.format, args.allow_empty, + blocked=False, stop_reason="completed", + pages_requested=1, pages_completed=1, + pages_failed=[], mode=args.mode, + source=SOURCE_DEFAULT, start_url=args.url, + final_url=args.url, + extra={"category": args.category + or category_from_url(args.url), + "via": "scraper_api"}) if state == "blocked": dump = f"{args.out}_scraperapi_debug.html" with open(dump, "w", encoding="utf-8") as f: @@ -293,7 +310,7 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: label = args.category or category_from_url(args.url) # A page the API SERVED that parses to nothing is this repo's bug, not - # an empty catalogue (CLAUDE.md §20). It used to return 4 — the same + # an empty catalogue. It used to return 4 — the same # conflation the engines carried until this audit — so it now reports a # non-complete stop reason, which `finish_run` maps to exit 5 and never # to "no jobs". @@ -349,7 +366,7 @@ def parse_args(): "selects.") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="wellfound_jobs_scraperapi", help="Output file prefix") - p.add_argument("--timeout", type=int, default=60, + p.add_argument("--timeout", type=page_flow.at_least_one, default=60, help=f"API-side task timeout in seconds (1-{MAX_API_TIMEOUT}, default 60)") p.add_argument("--cdp-url", default=None, help="Route the fetch through an existing browser session over CDP " @@ -366,11 +383,11 @@ def parse_args(): p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 products were parsed. Off by " "default so a failed fetch can't overwrite a good result.") - p.add_argument("--retries", type=int, default=1, + p.add_argument("--retries", type=page_flow.non_negative, default=1, help="Extra attempts if a bot-challenge page comes back. One retry is " "usually worth it. Each attempt is a separate billable task, so " "this defaults to 1.") - p.add_argument("--retry-delay", type=int, default=10, + p.add_argument("--retry-delay", type=page_flow.seconds, default=10, help="Seconds between retries (default 10)") p.add_argument("--dump-html", default=None, help="Also write the raw returned HTML to this path (always, even on success)") diff --git a/selenium_scraper.py b/selenium_scraper.py index e603485..ef578fb 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -551,8 +551,8 @@ def _solve_budget(args, spent: int): Measured 2026-09-17 from a datacenter address, which meets a real Cloudflare challenge on every fetch: ONE page bought THREE Turnstile - solves, and every token was refused. The cap read as enforced and was not - (CLAUDE.md §17). Both call sites now go through here. + solves, and every token was refused. The cap read as enforced and was not. + Both call sites now go through here. """ return spent < page_flow.SOLVES_PER_PAGE @@ -630,7 +630,11 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome "already made for it and SOLVES_PER_PAGE is %d.", page_num, solves_bought, page_flow.SOLVES_PER_PAGE) html = _snapshot(session, url) or "" - state = page_flow.classify(html, None, d["current_url"]()) + # Selenium exposes no HTTP status for a navigation, so the body alone + # decides here. `mode` is passed because a job page carries no + # listing payload: classified as a listing, every one of them came + # out `parse_error` and `--mode job` returned nothing. + state = page_flow.classify(html, None, d["current_url"](), mode=args.mode) # Wellfound server-renders its payload, so a listing is parseable in the # FIRST response and there is nothing to wait for on a healthy page. @@ -652,12 +656,13 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.info("Still nothing after %.0fs (%d match(es) for %s).", wait_ms / 1000.0, found, sel) html = _snapshot(session, url) or html - state = page_flow.classify(html, None, d["current_url"]()) + state = page_flow.classify(html, None, d["current_url"](), + mode=args.mode) - # The paid path is reached only for state "challenge" — Cloudflare's - # Managed Challenge, which IS a test. It is NOT reached for - # "blocked": the hard "You have been blocked" page carries no widget - # and no sitekey, so a solve there would be a charge for nothing. + # The paid path is reached only for a state the policy marks solvable + # ("blocked" here). Cloudflare's managed challenge IS a test; the + # branded refusal carries no widget and no sitekey, and the solver + # refuses to build a task without one, so it costs nothing there. # Bounded by SOLVES_PER_PAGE. Mirrors playwright_scraper. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): @@ -665,7 +670,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if handle_captcha_if_present(session, args): time.sleep(1) html = d["content"]() or html - state = page_flow.classify(html, url=d["current_url"]()) + state = page_flow.classify(html, None, d["current_url"](), + mode=args.mode) if state == "content": logger.info("The solve was accepted — page %d is content " "now.", page_num) @@ -676,7 +682,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if not page_flow.should_retry(state): # "content" and "empty" are both final answers. An empty page is - # a CORRECT one — a hub category has no grid — so retrying it + # a CORRECT one — a landing page that matched nothing — so retrying it # would re-confirm the same right answer, and rotating the exit # would blame an address for the URL it was given. break @@ -685,15 +691,29 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # so a different exit is the only thing that plausibly changes the # outcome. if block_attempt < block_retries: - logger.warning("Page %d came back as %s from %s — retrying from " - "another exit (%d/%d).", page_num, state, - mask(pool.current), block_attempt + 1, block_retries) - pool.advance(f"{state} on page {page_num}") - session.relaunch() - d = _driver(session) + if has_pool: + logger.warning("Page %d came back as %s from %s — retrying " + "from another exit (%d/%d).", page_num, state, + mask(pool.current), block_attempt + 1, + block_retries) + pool.advance(f"{state} on page {page_num}") + session.relaunch() + d = _driver(session) + else: + # No pool (no --proxy, or a single one), so there is no other + # exit to move to: re-fetch through the same access path, as + # the Playwright engine does. This branch used to touch + # `pool.current` unconditionally and raised AttributeError on + # the ordinary no-proxy block. + pause = args.retry_delay * (block_attempt + 1) + logger.warning("Page %d came back as %s — re-fetching through " + "the same access path in %.1fs (%d/%d).", + page_num, state, pause, block_attempt + 1, + block_retries) + time.sleep(pause) if load_failed: - # NAME the proxy when it was the proxy. CLAUDE.md §8: a dead exit and + # NAME the proxy when it was the proxy. A dead exit and # a timeout want opposite responses — another try at the same exit # versus a different exit — so a message that cannot tell them apart # leaves the reader guessing which they got. @@ -733,9 +753,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome "page and no key would help. What clears it, measured " "2026-09-17: an exit Wellfound does not score as a datacenter. Note " "this engine cannot use an authenticated remote CDP endpoint or " - "an authenticated proxy; see the README's engine limits — which " - "is why --mode profile is the one mode that needs a different " - "engine here. This is exit 3, distinct from a genuinely empty " + "an authenticated proxy (see the README's engine limits), so " + "use playwright_scraper.py for a residential exit with a " + "password. This is exit 3, distinct from a genuinely empty " "result (exit 4).", len(html or ""), references_own_assets(html or ""), debug_html) outcome.blocked_by = "cloudflare (hard block)" if html else "no-response" @@ -758,10 +778,9 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # page whose products have rendered guards nothing — and over # --cdp-endpoint the Scraping Browser's own auto-solve extension injects # such markers into every page it loads. - # Only for a state page_flow already counts as BLOCKED. An EMPTY page is - # a correct answer, and a live run of a /p/<slug> hub reported exit 3 on - # a page the site had plainly served because the hub's own performance - # script names `akamaihd.net`. Mirrors playwright_scraper exactly. + # Only for a state page_flow already counts as BLOCKED: an EMPTY page is + # a correct answer and must never become exit 3 because a marker + # matched it. Mirrors playwright_scraper exactly. vendor = (detect_bot_challenge(html, url=d["current_url"]()) if page_flow.counts_as_blocked(state) else None) if vendor: @@ -782,7 +801,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome if not page_flow.should_parse(state): # Dump it. A state that holds no rows is a state somebody will have # to diagnose, and the only way to tell a parsing bug from a page - # that never arrived is the exact bytes (CLAUDE.md §9). Until the + # that never arrived is the exact bytes. Until the # 2026-10-13 audit this path wrote nothing at all: the auditor's run # failed and left no artefact, which is half of why the cause was # misread. The blocked path has always dumped; this one did not. @@ -872,12 +891,11 @@ def scrape(args) -> int: outcomes: List[PageOutcome] = [] seen_keys = set() blocked = False - # All three modes are one row per business-at-a-location, so `sku` is the + # All three modes are one row per job posting, so `sku` (the job id) is the # key for all of them. dedupe_key = "sku" - # Only --mode profile is single-page. Both listing modes paginate - # identically, so neither may be treated as single-page — that is the - # silent-success failure this family exists to avoid. + # --mode job is one page by construction and --mode jobs is a feed + # served at one address; only --mode role paginates. stop_reason = ("single_page_mode" if args.mode == "job" else "single_page_listing" if args.mode == "jobs" else "completed") @@ -1054,20 +1072,20 @@ def parse_args(): help="Label to tag the run with in the sidecar. Defaults " "to what the URL selects. Wellfound has no category " "tree, so this names the QUERY, not a site concept.") - p.add_argument("--pages", type=int, default=1, + p.add_argument("--pages", type=page_flow.at_least_one, default=1, help="Listing pages to fetch (--mode role only). A run " "plans against the `pageCount` the site states on " "page 1 and never asks past it: page 48 of a 47-page " "listing answers HTTP 200 with page 1 AGAIN.") - p.add_argument("--delay", type=float, default=2.0, help="Delay between pages, seconds") - p.add_argument("--concurrency", type=int, default=1, metavar="N", + p.add_argument("--delay", type=page_flow.seconds, default=2.0, help="Delay between pages, seconds") + p.add_argument("--concurrency", type=page_flow.at_least_one, default=1, metavar="N", help="Accepted for flag parity and IGNORED here: parallel " "page fetching lives in playwright_scraper.py.") p.add_argument("--retries", type=page_flow.attempts, default=3, help="Attempts per page load before giving up (default 3). " "A page that comes back EMPTY is not retried: an empty " - "hub category is a correct answer, not a fault.") - p.add_argument("--retry-delay", type=float, default=2.0, + "landing page is a correct answer, not a fault.") + p.add_argument("--retry-delay", type=page_flow.seconds, default=2.0, help="Seconds before the first retry, doubling thereafter") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="wellfound_jobs", help="Output file prefix") @@ -1085,7 +1103,7 @@ def parse_args(): "Wins over --proxy.") p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run") p.add_argument("--proxy-shuffle", action="store_true") - p.add_argument("--proxy-block-retries", type=int, default=2) + p.add_argument("--proxy-block-retries", type=page_flow.non_negative, default=2) p.add_argument("--twocaptcha-key", default=None, help="2captcha.com API key") p.add_argument("--allow-empty", action="store_true", help="Write output files even when 0 rows were found.") @@ -1131,7 +1149,22 @@ def parse_args(): p.add_argument("--headless", action="store_true", default=True) p.add_argument("--headful", dest="headless", action="store_false") args = p.parse_args() + # What was TYPED, captured before the environment fills anything in. + typed_url = args.url + # Fill --twocaptcha-key / --cdp-endpoint / --proxy / --url from the + # environment or .env when the flag was not given. An explicit flag wins. env_config.apply(args) + # A WELLFOUND_URL from the environment or .env is a default target, not + # a query the user typed, so it yields to the role flags and to an + # explicit --mode job/jobs. Without this, a copied .env.example made + # `--role X` and `--mode jobs` exit 2 as if --url had been typed with them. + if not typed_url and args.url and ( + args.role or args.location or args.remote + or (args.mode in ("job", "jobs") + and mode_for_url(args.url) not in (None, args.mode))): + logger.info("Not using WELLFOUND_URL from the environment: the " + "command line selects the query.") + args.url = None # Spelled identically to playwright_scraper: --url and the query flags are # two ways to say the same thing, and only one may win. diff --git a/smoke_test.py b/smoke_test.py index 69ad291..559dc3f 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1,8 +1,8 @@ """ -smoke_test.py — the offline suite for wellfound-scraper. +Smoke_test.py — the offline suite for wellfound-scraper. One file of plain functions with inline fixtures. No pytest, no conftest, no -fixtures directory (CLAUDE.md §10); `tests/test_smoke.py` wraps this as a +fixtures directory; `tests/test_smoke.py` wraps this as a single pytest test so `pytest` works as an entry point without a second copy of the checks. @@ -98,7 +98,7 @@ def skip(group, reason): # # This fixture exists for one check and it is the most important one in the # file: `challenges.cloudflare.com` and `turnstile` are on this GOOD page and -# absent from both refusals above, so the marker CLAUDE.md §19 recommends is +# absent from both refusals above, so the marker other sites rely on is # inverted here. Counted on the real captures (2026-09-17): 1 and 6 on every # served landing page, 0 and 0 on both refusals. SERVED_WITH_SITE_TURNSTILE_HTML = LISTING_NEXT_DATA @@ -108,7 +108,7 @@ def skip(group, reason): # --------------------------------------------------------------------------- # The parser, asserted on VALUES rather than on coverage # --------------------------------------------------------------------------- -# A column can be 100% populated and entirely wrong (CLAUDE.md §10), so every +# A column can be 100% populated and entirely wrong, so every # check below pins a figure from the real capture rather than counting # non-nulls. @@ -238,7 +238,7 @@ def check_salary_period_is_null_on_a_listing_row(): """The listing string carries no period, so the column must not invent one. `$140k` on a job board reads as a year, but the PAGE does not say so. - CLAUDE.md §8: never present a guess as a fact. The detail page does state + Never present a guess as a fact. The detail page does state one, and the next check pins that it is read. """ import product_parser as P @@ -278,8 +278,8 @@ def check_feed_uses_different_types_and_one_parser_reads_both(): def check_detail_page_is_a_different_mechanism(): """A job page has no `__NEXT_DATA__` and one schema.org JobPosting. - This is CLAUDE.md §20's "a DETAIL page may publish a different JSON-LD - type than the listing" in its sharpest form: a different MECHANISM. The + A DETAIL page may publish a different JSON-LD type than the listing, + here in its sharpest form: a different MECHANISM. The failure it guards is silent — the listing parser returns [] rather than raising — so it is pinned directly, in both directions. """ @@ -313,8 +313,8 @@ def check_detail_page_is_a_different_mechanism(): def check_the_two_sources_agree_about_pay(): """Where both views publish a salary they must agree. - CLAUDE.md §4 says to check whether structured and displayed prices - disagree on every new site. Here they do not: job 4697947 is + Structured and displayed prices are checked for disagreement on every + new site. Here they do not: job 4697947 is `'$100k – $180k • 0.05% – 0.25%'` on the listing and 100000-180000 USD in the detail page's `baseSalary`. That agreement is what lets `--mode job` enrich a `--mode role` run instead of contradicting it, and it is pinned @@ -343,7 +343,7 @@ def check_an_out_of_range_page_is_detected_from_the_payload(): A run that trusted its own request would collect page 1 for as long as it was asked to and report success. The server states which page it really used inside the Apollo cache key, and that is an unambiguous signal - rather than the weaker "no new sku" heuristic (CLAUDE.md §17). + rather than the weaker "no new sku" heuristic. """ import product_parser as P page = P.parse_listing_page(LISTING_NEXT_DATA, page=48) @@ -378,7 +378,7 @@ def check_only_the_landing_routes_are_addressable(): `?page=2` on /jobs does not fail and does not empty — it returns the identical 46 job ids as page 1. A `page_url()` used unconditionally there would find no new sku, conclude the listing was exhausted, and report a - COMPLETE run holding page 1 (CLAUDE.md §18). + COMPLETE run holding page 1. """ import page_flow for url, want in ( @@ -444,7 +444,7 @@ def check_route_and_mode_are_derived_from_the_url(): # Wellfound's own robots.txt, fetched 2026-10-13 through a residential exit # (it answers 403 to a datacentre address, like every other route here). -# Inline rather than in a fixtures directory, per CLAUDE.md §10. +# Inline rather than in a fixtures directory, like every fixture here. ROBOTS_TXT = """# # Hey human, # @@ -531,7 +531,7 @@ def check_every_route_this_scraper_reads_is_robots_allowed(): path) # The guard must have had something to read, or it passes for the wrong - # reason (CLAUDE.md §22): an empty robots file allows everything. + # reason: an empty robots file allows everything. rules = [l for l in ROBOTS_TXT.splitlines() if l.lower().startswith("disallow")] check("...and the fixture really carries rules", len(rules) > 10, @@ -539,7 +539,7 @@ def check_every_route_this_scraper_reads_is_robots_allowed(): def check_unsupported_urls_are_refused_with_a_reason(): - """A wrong reason sends the reader hunting for a typo (CLAUDE.md §5).""" + """A wrong reason sends the reader hunting for a typo.""" import product_parser as P ok, why = P.is_supported_url("https://wellfound.com/role/r/software-engineer") check("supported: a landing page is accepted", ok, why) @@ -595,7 +595,7 @@ def check_page_states_on_real_captures(): equal("state: 404 is not a block", P.detect_page_state("<html></html>", 404), "not_found") # A served page whose payload we could not read is OUR bug, never - # "0 jobs" (CLAUDE.md §20). + # "0 jobs". broken = ('<html><body><img src="https://photos.wellfound.com/x.jpg">' '<script id="__NEXT_DATA__" type="application/json">' '{"props":{"pageProps":{"apolloState":{"data":{"ROOT_QUERY":{}}}}}}' @@ -605,7 +605,7 @@ def check_page_states_on_real_captures(): def check_markers_do_not_match_a_page_wellfound_serves(): - """Count every marker on a page known GOOD before trusting it (§18). + """Count every marker on a page known GOOD before trusting it. This is the check that caught the mistake this repo would otherwise have shipped. `challenges.cloudflare.com` is the marker the rest of this @@ -652,7 +652,7 @@ def check_only_one_refusal_skin_carries_a_widget(): "Just a moment..." (browser) _cf_chl_opt 7x, turnstile 1x So the no-JS skin carries **no widget at all**, and that is the precise - sense in which a page is unsolvable (CLAUDE.md §19: "unsolvable" is a + sense in which a page is unsolvable ("unsolvable" is a property of a PAGE with nothing on it, never of a vendor). Paying for a task built from a page like that buys an ERROR_CAPTCHA_UNSOLVABLE, which is why the engines refuse to build one without a captured sitekey. @@ -680,7 +680,7 @@ def check_only_one_refusal_skin_carries_a_widget(): def check_the_sites_own_turnstile_is_recognised_as_configured(): """"Did we meet a captcha" is the wrong question; "is one configured, and - would we recognise it" is the right one (CLAUDE.md §18). + would we recognise it" is the right one. One is: Wellfound publishes its sitekey in its own page config and renders a widget with it when its own fetch wrapper is challenged. That @@ -731,7 +731,7 @@ def check_company_size_tokens(): def check_content_not_obtained_is_never_an_empty_result(): - """A page we never read must not report "0 jobs" (CLAUDE.md §9, §25). + """A page we never read must not report "0 jobs". Found by a third-party audit on 2026-10-13. The chain was four links long and every one of them looked right on its own: `unknown` -> @@ -878,7 +878,7 @@ def check_state_policy(): page_flow.should_retry("blocked") and page_flow.should_solve("blocked") and page_flow.counts_as_blocked("blocked")) - # A served page we could not read is OUR bug, never "0 jobs" (§20), so + # A served page we could not read is OUR bug, never "0 jobs", so # it neither counts as blocked nor buys a solve. check("parse_error: retried once, never solved, NOT blocked", page_flow.should_retry("parse_error") @@ -901,7 +901,7 @@ def check_state_policy(): def check_policy_constants_have_a_consumer(): - """§17: a policy constant nothing reads is the same defect as dead code. + """A policy constant nothing reads is the same defect as dead code. `RETRY_ON_BLOCKED` carried a paragraph of justification in a sibling repo and no engine consulted it, so setting it False changed nothing. @@ -935,7 +935,7 @@ def check_policy_constants_have_a_consumer(): def check_row_schema(): from output_writer import JobPosting, ROW_CLASS_BY_MODE, UNIQUE_BY_SKU_MODES names = [f.name for f in fields(JobPosting)] - equal("the family prefix is byte-identical and in order (§9)", + equal("the family prefix is byte-identical and in order", names[:5], ["source", "scraped_at", "url", "sku", "title"]) for gone in ("price", "currency", "original_price", "discount_pct", "in_stock", "brand"): @@ -1026,7 +1026,7 @@ def check_exit_codes(): def check_a_partial_run_says_so_and_still_writes(): - """§9: blocked != empty != partial, and all three are distinct exit codes. + """Blocked != empty != partial, and all three are distinct exit codes. Only the CONSTANT was pinned — `EXIT_PARTIAL == 6` — never the behaviour, which is the half a consumer branches on. A partial run has to do four @@ -1088,7 +1088,7 @@ def check_a_run_that_finds_nothing_writes_nothing(): code = save([], prefix, "json", allow_empty=True) equal("--allow-empty WRITES the empty file...", json.load(open(prefix + ".json", encoding="utf-8")), []) - # ...and still reports exit 4. Pinned deliberately (§10: pin a known + # ...and still reports exit 4. Pinned deliberately (pin a known # behaviour rather than half-guarding it): "zero businesses" is true # whether or not the file was written, and a caller that wanted the # file still wants to know the result was empty. @@ -1133,7 +1133,7 @@ def check_sidecar_shape(): # --------------------------------------------------------------------------- -# The engines — the five checks CLAUDE.md §17 says to steal +# The engines — binding, flags, names, imports and numbers # --------------------------------------------------------------------------- ENGINES = ("playwright_scraper", "selenium_scraper", "puppeteer_scraper") @@ -1180,7 +1180,7 @@ def check_engines_import_their_driver_at_module_level(): def check_shared_calls_bind_against_the_real_signature(): - """§17's check #1, and the one that earns its keep. + """Shared calls bind against the callee, and the one that earns its keep. A sibling repo shipped `classify(html, url=…)` in two of three engines against a callee taking `status` second, and BOTH crashed on their first @@ -1296,7 +1296,7 @@ def _argparse_flags(module_name): return flags -# The family's flag contract (CLAUDE.md §9), plus this repo's own additions. +# The family's flag contract, plus this repo's own additions. CONTRACT_FLAGS = { "--url", "--pages", "--category", "--format", "--out", "--delay", "--retries", "--retry-delay", "--concurrency", "--proxy", "--proxy-file", @@ -1312,7 +1312,7 @@ def _argparse_flags(module_name): def check_engine_flag_sets(): - """§17's check #2: against the contract AND against each other, both ways. + """Flag sets: against the contract AND against each other, both ways. A missing flag fails; so does closing a difference the README documents. """ @@ -1329,7 +1329,7 @@ def check_engine_flag_sets(): # and could not launch it on the development machine, so it needs a way # to point at another one. Its twins have no equivalent because they do # not ship a browser. Listed here so that closing the difference — or - # growing a second one — fails the build (§17). + # growing a second one — fails the build. DOCUMENTED_DIFFERENCES = {"puppeteer_scraper": {"--chromium-path"}} names = sorted(sets) for i in range(len(names) - 1): @@ -1352,8 +1352,7 @@ def check_banned_and_removed_flags(): Wellfound serves one host in one language, so such a flag could only contradict what the URL already says. On `fingerprint_client.py` the same name is legitimate — there it picks a fingerprint locale, not a target — - which is why this check is scoped to the engines rather than to the tree - (CLAUDE.md §10). + which is why this check is scoped to the engines rather than to the tree. What the rule is really about is a flag that can disagree with the URL, and the equivalent on this site IS enforced: `--role`, `--location` and @@ -1375,7 +1374,7 @@ def check_banned_and_removed_flags(): def check_undefined_names_in_every_module(): - """§10: compileall proves a file PARSES, not that its names RESOLVE. + """Compileall proves a file PARSES, not that its names RESOLVE. A live run of a sibling repo's pyppeteer engine died with NameError on a line reached only while fetching, after an import had been removed — the @@ -1433,7 +1432,7 @@ def _import_graph(entrypoint): def check_dockerfile_copies_everything_the_entrypoint_imports(): - """§10: all three repos in this family shipped an image that died with + """All three repos in this family shipped an image that died with ModuleNotFoundError on every invocation, --help included, because proxy_pool.py was missing from the COPY list. CI never built the image; this check needs no Docker.""" @@ -1481,7 +1480,7 @@ def check_env_example_documents_exactly_what_the_loader_reads(): def check_a_copied_env_example_reads_as_UNSET(): - """§17: `cp .env.example .env` followed by a run must not connect. + """`cp .env.example .env` followed by a run must not connect. The placeholder check was a literal set in a sibling repo, and the two credentialled URLs are documented the way the vendor documents them — @@ -1496,14 +1495,20 @@ def check_a_copied_env_example_reads_as_UNSET(): return text = open(example, encoding="utf-8").read() values = dict(re.findall(r"^([A-Z][A-Z0-9_]+)=(.*)$", text, re.M)) - check("the example actually sets every variable", - set(values) == set(env_config.ENV_KEYS), + # WELLFOUND_URL is shipped COMMENTED OUT, and that is pinned: an active + # one filled --url for every run of a copied example, and the builder + # flags then exited 2 as if a URL had been typed beside them. The engines + # now also ignore an environment URL when the role flags are given, so + # this is belt and braces rather than the only defence. + check("WELLFOUND_URL is an example, not an active setting", + "WELLFOUND_URL" not in values + and re.search(r"^#\s*WELLFOUND_URL=https://wellfound\.com/", text, + re.M) is not None) + check("the example actually sets every credential variable", + set(values) == set(env_config.ENV_KEYS) - {"WELLFOUND_URL"}, "example has %s, loader reads %s" % (sorted(values), sorted(env_config.ENV_KEYS))) - # Every CREDENTIAL must read as unset. The default TARGET must not: it is - # a real, usable URL, and blanking it would remove the one setting this - # file exists to make convenient (§17's check #3 says exactly this — the - # credentials unset, the non-credential default still usable). + # Every CREDENTIAL must read as unset. CREDENTIALS = {"TWOCAPTCHA_KEY", "WELLFOUND_CDP_ENDPOINT", "WELLFOUND_PROXY"} before = dict(os.environ) @@ -1535,7 +1540,7 @@ def check_a_copied_env_example_reads_as_UNSET(): def check_credential_scan_is_one_implementation_invoked_from_both(): - """§17: two sources of truth, one dead and one holed. + """Two sources of truth, one dead and one holed. `.github/ci_checks.py` sat in three repos invoked by NOTHING, while tests.yml carried an inline grep doing a narrower version of the same job @@ -1552,7 +1557,7 @@ def check_credential_scan_is_one_implementation_invoked_from_both(): # file being missing: two suites in this family run INSIDE the Docker # image, which deliberately COPYs no .github/, and a check that quietly # starts passing once its input disappears is the same failure this - # function is about (CLAUDE.md §22). + # function is about. if not os.path.isdir(workflow_dir): skip("ci-wiring", "no .github/ in this tree (the Docker image)") elif os.path.exists(workflow): @@ -1581,7 +1586,7 @@ def check_credential_scan_is_one_implementation_invoked_from_both(): "tests.yml's offline job mentions %r — one implementation, " "in ci_checks.py, invoked from both" % marker) # And the guard must have had something to read, or it passed for the - # wrong reason (CLAUDE.md §22). + # wrong reason. check("...and the offline job was actually found to scan", "ci_checks.py" in offline, "no offline job in tests.yml") result = subprocess.run([sys.executable, script, "--all"], cwd=HERE, @@ -1609,7 +1614,7 @@ def check_the_credential_scan_survives_a_venv_in_the_tree(): The fix is structural rather than a longer list of names — a directory holding `pyvenv.cfg` is a virtualenv whatever it is called — and this pins BOTH halves, because narrowing a credential scan is exactly how one - stops catching things. CLAUDE.md §22 records a sibling repo whose scan + stops catching things. A sibling repo's scan caught an UNTRACKED `.env.bak` holding a live key, so scanning must not be reduced to tracked files. """ @@ -1648,7 +1653,7 @@ def check_the_credential_scan_survives_a_venv_in_the_tree(): def check_banned_wording(): - """§12: enforced by this test rather than by review.""" + """Enforced by this test rather than by review.""" for root, dirs, files in os.walk(HERE): dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", ".pytest_cache", "node_modules")] @@ -1666,7 +1671,7 @@ def check_banned_wording(): def check_concurrency_with_the_browser_stubbed(): - """§10: a live run cannot always reach this machinery. + """A live run cannot always reach this machinery. Page 1 is fetched alone and decides how many pages there are, so a blocked page 1 means the workers never start. Driven directly instead, @@ -1812,7 +1817,7 @@ def check_worker_pools_start_on_different_exits(): def check_fingerprint_kwargs_are_ones_the_driver_accepts(): - """§10: an unknown key in new_context(**kwargs) is a TypeError at launch, + """An unknown key in new_context(**kwargs) is a TypeError at launch, on the PAID path, at runtime.""" engine = _import_engine("playwright_scraper") if engine is None: @@ -1867,7 +1872,7 @@ def check_every_engine_exposes_the_same_public_surface(): def check_every_solve_is_counted_against_the_budget(): """`SOLVES_PER_PAGE` is a MONEY limit, so every call that can buy must be - counted — CLAUDE.md §17's "a policy constant nothing reads". + counted — a policy constant nothing reads is not enforced. `handle_captcha_if_present` is called TWICE per attempt in every engine: once before the page is classified (so a challenge is cleared before @@ -1908,7 +1913,7 @@ def check_every_solve_is_counted_against_the_budget(): def check_a_dead_proxy_is_reported_as_a_proxy_failure(): - """CLAUDE.md §8: a proxy failure is not a timeout, and the two want + """A proxy failure is not a timeout, and the two want opposite responses — another try at the same exit versus a different one. The engines all compute the reason (`_proxy_failure`) and, WITH a pool, @@ -1931,7 +1936,7 @@ def check_a_dead_proxy_is_reported_as_a_proxy_failure(): # Anchored on the GIVE-UP branch — the one that reports and returns — # not on the `if load_failed: break` inside the retry loop, which # comes first in the file and would make this check read the wrong - # block and pass for the wrong reason (CLAUDE.md §22). + # block and pass for the wrong reason. marker = " outcome.load_failed = True" if marker not in source: check("%s has a give-up branch to check" % module, False) @@ -1963,7 +1968,7 @@ class _E(Exception): def check_engines_do_not_evaluate_a_string_in_the_browser(): - """§18: a site whose CSP omits `unsafe-eval` kills wait_for_function with + """A site whose CSP omits `unsafe-eval` kills wait_for_function with an EvalError and takes the run down with exit 1. Wellfound has not been measured for that, and the cheap habit costs nothing where it would have been allowed.""" @@ -1981,7 +1986,7 @@ def check_engines_do_not_evaluate_a_string_in_the_browser(): def check_credentials_never_reach_a_log(): - """§8: an EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. + """An EXCEPTION MESSAGE is a log, and the masker must be GLOBAL. A Playwright connection error repeats the endpoint five times (the message plus a four-line call log), so a masker handling only the first @@ -2027,7 +2032,7 @@ def check_sample_output_matches_the_schema(): def check_readme_numbers_are_not_stale(): - """§17's check #4: diff every numeric claim against what is on disk. + """Diff every numeric claim against what is on disk. Only the figures that MUST hold are pinned: a number that legitimately varies between runs is written as a range in the README and not checked @@ -2067,7 +2072,7 @@ def _tree_state(): def check_no_test_mutates_the_working_tree(): - """§10: one suite used its own file as a fake chromedriver and chmod'd it + """One suite used its own file as a fake chromedriver and chmod'd it to 755, leaving a mode change in git status. Compares the tree against how it looked when the suite STARTED, not @@ -2085,7 +2090,7 @@ def check_no_test_mutates_the_working_tree(): def check_captcha_capability_claims_match_the_code(): - """§19: the most expensive bug this family can ship is a SENTENCE. + """The most expensive bug this family can ship is a SENTENCE. It fails in both directions and this family has shipped both: @@ -2254,6 +2259,467 @@ def _post(url, headers=None, json=None, timeout=None, **_kw): # noqa: A002 status == 403 and isinstance(status, int)) + +# --------------------------------------------------------------------------- +# The engines' page loop, driven with the driver library STUBBED +# --------------------------------------------------------------------------- +# The engine groups above skip when Playwright, Selenium or pyppeteer is not +# installed, which is every core-only run, and that is how two defects in the +# mirror engines survived a green suite: Selenium and pyppeteer raised +# AttributeError on the ordinary no-proxy block (`pool.current` on None), and +# both classified every `--mode job` page as a listing. These checks load each +# engine from disk with its driver library replaced by stubs, so they run in +# the core venv and in every engine venv alike, and drive the REAL +# `_fetch_one_page` with a fake session. + +_STUB_DRIVER_MODULES = ( + "playwright", "playwright.sync_api", + "selenium", "selenium.common", "selenium.common.exceptions", + "selenium.webdriver", "selenium.webdriver.chrome", + "selenium.webdriver.chrome.options", "selenium.webdriver.common", + "selenium.webdriver.common.by", "selenium.webdriver.support", + "selenium.webdriver.support.ui", + "pyppeteer", +) + + +def _engine_with_stub_driver(name): + """Import `<name>.py` under a private module name with every driver + library replaced by a stub whose attributes are exception classes. + sys.modules is restored afterwards, so the real engine imports elsewhere + in this suite are unaffected.""" + import importlib.util + import types + + class _StubLib(types.ModuleType): + def __getattr__(self, attr): + if attr.startswith("__"): + raise AttributeError(attr) + cls = type(attr, (Exception,), {}) + setattr(self, attr, cls) + return cls + + saved = {m: sys.modules.get(m) for m in _STUB_DRIVER_MODULES} + try: + for m in _STUB_DRIVER_MODULES: + sys.modules[m] = _StubLib(m) + for m in _STUB_DRIVER_MODULES: + if "." in m: + parent, child = m.rsplit(".", 1) + setattr(sys.modules[parent], child, sys.modules[m]) + spec = importlib.util.spec_from_file_location( + "_stubbed_" + name, os.path.join(HERE, name + ".py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + finally: + for m, v in saved.items(): + if v is None: + sys.modules.pop(m, None) + else: + sys.modules[m] = v + + +class _Anything: + """A driver object that accepts any call and returns None.""" + def __getattr__(self, attr): + if attr.startswith("__"): + raise AttributeError(attr) + return lambda *a, **k: None + + +class _FakeResponse: + def __init__(self, status): + self.status = status + + +def _drive_one_page(engine_name, html, mode="role", pool=None, timeout=False, + status=200, url=None): + """Run the engine's real `_fetch_one_page` once against a fake session. + Returns (outcome or None, exception or None, calls).""" + import argparse as _argparse + engine = _engine_with_stub_driver(engine_name) + url = url or ("https://wellfound.com/jobs/4697947-senior-software-engineer" + if mode == "job" + else "https://wellfound.com/role/r/software-engineer") + calls = {"goto": 0, "relaunch": 0, "solve": 0} + tmp = tempfile.mkdtemp() + + def navigate(*_a, **_k): + calls["goto"] += 1 + if timeout: + raise timeout_exc("Navigation timeout of 60000 ms exceeded") + return _FakeResponse(status) + + page = _Anything() + page.url = url + page.goto = navigate + page.query_selector_all = lambda *_a: [] + session = _Anything() + session.page = page + + def relaunch(): + calls["relaunch"] += 1 + session.relaunch = relaunch + + if engine_name == "playwright_scraper": + timeout_exc = engine.PWTimeout + engine._snapshot = lambda _page, _url: html + elif engine_name == "selenium_scraper": + timeout_exc = engine.TimeoutException + driver = _Anything() + driver.get = navigate + driver.current_url = url + session.driver = driver + engine._snapshot = lambda _s, _url: html + engine._driver = lambda _s: { + "count": lambda _sel: 0, "sleep": lambda _ms: None, + "content": lambda: html, "current_url": lambda: url} + else: + timeout_exc = RuntimeError + bridge = _Anything() + bridge.run = lambda value: value + session.bridge = bridge + engine._snapshot = lambda _s, _url: html + engine._content = lambda _s: html + engine._driver = lambda _s: { + "count": lambda _sel: 0, "sleep": lambda _ms: None, + "content": lambda: html, "current_url": lambda: url} + + def no_solve(*_a, **_k): + calls["solve"] += 1 + return False + engine.handle_captcha_if_present = no_solve + + args = _argparse.Namespace( + url=url, mode=mode, pages=1, retries=1, retry_delay=0, delay=0, + proxy_block_retries=1, out=os.path.join(tmp, "out"), dump_html=None, + solve_captcha="when-blocked", twocaptcha_key=None, captcha_api="v2", + min_score=0.7, cdp_endpoint=None, proxy=None, format="json", + allow_empty=False, category=None) + import logging + import shutil + logging.disable(logging.CRITICAL) + try: + return engine._fetch_one_page(session, args, pool, 1, url), None, calls + except Exception as e: # noqa: BLE001 — the point is to catch it + return None, e, calls + finally: + logging.disable(logging.NOTSET) + shutil.rmtree(tmp, ignore_errors=True) + + +def check_mirror_engines_survive_a_block_and_a_timeout_without_a_proxy(): + """No proxy is the ordinary case, and a block is the ordinary outcome + from a datacentre address. Every engine must report it, not crash.""" + from proxy_pool import ProxyPool + for engine_name in ENGINES: + for label, html in (("branded refusal", BRANDED_BLOCK_HTML), + ("interstitial", INTERSTITIAL_HTML)): + outcome, err, calls = _drive_one_page(engine_name, html) + check("%s: a %s with no proxy is reported, not raised" + % (engine_name, label), err is None, repr(err)) + if outcome is not None: + equal("%s: ...as state blocked" % engine_name, + outcome.state, "blocked") + equal("%s: ...after one re-fetch through the same path" + % engine_name, calls["goto"], 2) + equal("%s: ...without relaunching anything" % engine_name, + calls["relaunch"], 0) + outcome, err, calls = _drive_one_page(engine_name, "", timeout=True) + check("%s: a load timeout with no proxy is reported, not raised" + % engine_name, err is None, repr(err)) + if outcome is not None: + check("%s: ...as a load failure" % engine_name, + outcome.load_failed) + # With a real pool the block retry DOES rotate. + pool = ProxyPool(["http://a:1", "http://b:2"], rotate="per-run") + outcome, err, calls = _drive_one_page(engine_name, BRANDED_BLOCK_HTML, + pool=pool) + check("%s: with a two-exit pool a block rotates and relaunches" + % engine_name, err is None and calls["relaunch"] == 1, + "err=%r relaunch=%r" % (err, calls["relaunch"])) + + +def check_every_engine_reads_a_job_page_as_a_job(): + """`--mode job` must reach the job reader in every engine.""" + for engine_name in ENGINES: + outcome, err, _calls = _drive_one_page(engine_name, JOB_DETAIL_HTML, + mode="job") + check("%s: a job page in --mode job does not raise" % engine_name, + err is None, repr(err)) + if outcome is not None: + equal("%s: ...is classified content" % engine_name, + outcome.state, "content") + equal("%s: ...and yields its one row" % engine_name, + len(outcome.products), 1) + + +def check_every_classify_call_passes_the_mode(): + """Signature binding cannot see an omitted OPTIONAL argument, and an + omitted `mode` silently classifies every job page as a listing.""" + for engine_name in ENGINES: + tree = ast.parse(open(os.path.join(HERE, engine_name + ".py"), + encoding="utf-8").read()) + sites = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = (func.attr if isinstance(func, ast.Attribute) + else getattr(func, "id", None)) + if name not in ("classify", "_classify"): + continue + if isinstance(func, ast.Attribute) and not ( + isinstance(func.value, ast.Name) + and func.value.id == "page_flow"): + continue + # page_flow.classify(html, status, url, mode) / _classify(page, + # html, status, mode): mode is the 4th positional or a keyword. + passes = (any(kw.arg == "mode" for kw in node.keywords) + or len(node.args) >= 4) + sites.append((node.lineno, passes)) + check("%s has classify call sites" % engine_name, len(sites) >= 3, + "%d found" % len(sites)) + for lineno, passes in sites: + check("%s:%d classify(...) passes mode" % (engine_name, lineno), + passes) + + +def check_the_http_status_reaches_the_classifier(): + """A 404 is `not_found` only if the status gets there. Playwright and + pyppeteer have one; Selenium does not expose it and decides by body.""" + gone = "<html><head><title>Page not found" + for engine_name in ("playwright_scraper", "puppeteer_scraper"): + outcome, err, calls = _drive_one_page(engine_name, gone, mode="job", + status=404) + check("%s: a 404 does not raise" % engine_name, err is None, repr(err)) + if outcome is not None: + equal("%s: ...is classified not_found" % engine_name, + outcome.state, "not_found") + equal("%s: ...and is not re-fetched" % engine_name, + calls["goto"], 1) + outcome, err, _calls = _drive_one_page(engine_name, JOB_DETAIL_HTML, + mode="job", status=200) + check("%s: a 200 job page is still content" % engine_name, + outcome is not None and outcome.state == "content", + "err=%r state=%r" % (err, getattr(outcome, "state", None))) + + +def check_an_environment_url_does_not_block_the_builder_flags(): + """`cp .env.example .env` used to make `--role X` exit 2: the URL from + the environment was judged as if it had been typed beside the flag.""" + before_env, before_argv = dict(os.environ), list(sys.argv) + try: + os.environ["WELLFOUND_URL"] = "https://wellfound.com/role/r/software-engineer" + for engine_name in ENGINES: + engine = _engine_with_stub_driver(engine_name) + + def run(argv): + sys.argv = [engine_name + ".py"] + argv + try: + return engine.parse_args(), None + except SystemExit as e: + return None, e.code + + args, code = run(["--role", "data-scientist"]) + check("%s: env URL + --role builds the role URL" % engine_name, + args is not None + and args.url == "https://wellfound.com/role/data-scientist", + "exit %r, url %r" % (code, getattr(args, "url", None))) + args, code = run(["--role", "sales", "--location", "london"]) + check("%s: env URL + --role --location builds that URL" + % engine_name, + args is not None and "/role/l/sales/london" in args.url, + "exit %r" % code) + args, code = run(["--mode", "jobs"]) + check("%s: env URL + --mode jobs reads the feed" % engine_name, + args is not None and args.url.rstrip("/").endswith("/jobs"), + "exit %r, url %r" % (code, getattr(args, "url", None))) + args, code = run([]) + check("%s: with no flags the env URL is still the default" + % engine_name, + args is not None + and args.url == os.environ["WELLFOUND_URL"], + "exit %r" % code) + args, code = run(["--url", "https://wellfound.com/role/r/sales", + "--role", "sales"]) + equal("%s: a TYPED --url with --role is still refused" + % engine_name, code, 2) + finally: + os.environ.clear() + os.environ.update(before_env) + sys.argv = before_argv + + +def check_counting_flags_refuse_values_that_would_skip_or_crash(): + """A negative block-retry count made the loop run zero times (no fetch + at all), and a negative delay crashed in time.sleep after page 1.""" + before_argv = list(sys.argv) + bad = [("--pages", "0"), ("--pages", "-1"), ("--concurrency", "0"), + ("--proxy-block-retries", "-1"), ("--delay", "-1"), + ("--retry-delay", "-0.5"), ("--retries", "0")] + try: + for engine_name in ENGINES: + engine = _engine_with_stub_driver(engine_name) + for flag, value in bad: + sys.argv = [engine_name, "--url", + "https://wellfound.com/role/r/software-engineer", + flag, value] + err = io.StringIO() + real_err, sys.stderr = sys.stderr, err + try: + engine.parse_args() + code = None + except SystemExit as e: + code = e.code + finally: + sys.stderr = real_err + check("%s %s %s exits 2 naming the flag" + % (engine_name, flag, value), + code == 2 and ("argument " + flag) in err.getvalue(), + "exit %r: %s" % (code, err.getvalue()[-160:])) + sys.argv = [engine_name, "--url", + "https://wellfound.com/role/r/software-engineer", + "--proxy-block-retries", "0", "--delay", "0", + "--pages", "1"] + try: + ok = engine.parse_args() is not None + except SystemExit: + ok = False + check("%s accepts 0 block retries and a 0 delay" % engine_name, ok) + finally: + sys.argv = before_argv + + +def check_csv_neutralises_formula_cells_and_json_does_not(): + from output_writer import JobPosting, finish_run + hostile = JobPosting(title="=HYPERLINK(\"http://x\")", company_name="@corp", + sku="1", url="https://wellfound.com/jobs/1-x", + description="-flat", salary_min=-5) + plain = JobPosting(title="Engineer", company_name="Acme", sku="2", + url="https://wellfound.com/jobs/2-y") + with tempfile.TemporaryDirectory() as tmp: + prefix = os.path.join(tmp, "run") + real_out, sys.stdout = sys.stdout, io.StringIO() + try: + rc = finish_run([hostile, plain], prefix, "both", False, + blocked=False, stop_reason="completed", + pages_requested=1, pages_completed=1, + start_url="u", final_url="u", mode="role", + extra={"category": "c"}) + finally: + sys.stdout = real_out + equal("a run with formula-shaped text still exits 0", rc, 0) + rows = list(csv.DictReader(open(prefix + ".csv", newline="", + encoding="utf-8"))) + equal("CSV: a leading = is prefixed", rows[0]["title"], + "'=HYPERLINK(\"http://x\")") + equal("CSV: a leading @ is prefixed", rows[0]["company_name"], "'@corp") + equal("CSV: a leading - in text is prefixed", rows[0]["description"], + "'-flat") + equal("CSV: a NUMBER is never prefixed", rows[0]["salary_min"], "-5") + equal("CSV: ordinary text untouched", rows[1]["title"], "Engineer") + data = json.load(open(prefix + ".json", encoding="utf-8")) + equal("JSON keeps the text exactly", data[0]["title"], + "=HYPERLINK(\"http://x\")") + meta = json.load(open(prefix + ".meta.json", encoding="utf-8")) + equal("the sidecar counts the escaped cells", + meta.get("csv_cells_escaped"), 3) + equal("...beside the caller's own extra", meta.get("category"), "c") + real_out, sys.stdout = sys.stdout, io.StringIO() + try: + finish_run([plain], prefix, "json", False, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u", + mode="role", extra={"csv_cells_escaped": "caller"}) + finally: + sys.stdout = real_out + meta = json.load(open(prefix + ".meta.json", encoding="utf-8")) + check("a JSON-only run writes no escape count of its own", + meta.get("csv_cells_escaped") == "caller", + repr(meta.get("csv_cells_escaped"))) + + +def check_the_credential_scan_is_judged_relative_to_the_repo(): + """A clone under ~/build, /tmp/venv or any skipped NAME used to scan 0 + files and print ok, because exclusions were tested on the absolute path.""" + import importlib.util + script = os.path.join(HERE, ".github", "ci_checks.py") + if not os.path.exists(script): + skip("credential-scan", "no .github/ in this tree (the Docker image)") + return + planted = "ab" * 16 # 32 hex characters, assembled at run time + for parent in ("venv", "build", ".venv", "__pycache__", "tmp", "plain"): + with tempfile.TemporaryDirectory() as tmp: + repo = pathlib.Path(tmp) / parent / "wf" + repo.mkdir(parents=True) + (repo / "probe.py").write_text("KEY = '%s'\n" % planted) + spec = importlib.util.spec_from_file_location("_ci_rel", script) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.REPO = repo + real_out, sys.stdout = sys.stdout, io.StringIO() + try: + failed = mod.secret_check() + finally: + sys.stdout = real_out + check("a clone under %s/ still finds a planted key" % parent, + any("probe.py" in f for f in failed), repr(failed)) + with tempfile.TemporaryDirectory() as tmp: + spec = importlib.util.spec_from_file_location("_ci_rel", script) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod.REPO = pathlib.Path(tmp) + real_out, sys.stdout = sys.stdout, io.StringIO() + try: + failed = mod.secret_check() + finally: + sys.stdout = real_out + check("a scan that read 0 files FAILS instead of passing", + bool(failed), repr(failed)) + + +# Private working notes must not be cited from public files. Assembled from +# pieces so this file does not match itself. +_PRIVATE_CITATIONS = ( + "CLAUDE" + ".md", "claude_" + "2scraper", "2scraper_" + "repo_rules", + "play" + "book", "template " + "\N{SECTION SIGN}", +) +_SECTION_SIGN = "\N{SECTION SIGN}" + + +def check_no_private_citations_in_public_files(): + """Every tracked file is public. A reference to a private instruction + file, or a bare section number from one, means nothing to a reader and + leaks the existence of notes they cannot see.""" + try: + tracked = subprocess.run(["git", "ls-files"], cwd=HERE, + capture_output=True, text=True, + check=True).stdout.split() + except (OSError, subprocess.CalledProcessError): + tracked = [] + for root, dirs, files in os.walk(HERE): + dirs[:] = [d for d in dirs if d not in (".git", "__pycache__")] + tracked += [os.path.relpath(os.path.join(root, f), HERE) + for f in files] + scanned = 0 + for rel in tracked: + path = os.path.join(HERE, rel) + try: + text = open(path, encoding="utf-8").read() + except (OSError, UnicodeDecodeError): + continue + scanned += 1 + for token in _PRIVATE_CITATIONS + (_SECTION_SIGN,): + if token in text: + line = text[:text.index(token)].count("\n") + 1 + check("%s:%d cites no private notes (%r)" % (rel, line, token), + False) + check("the private-citation scan read the tracked files (%d)" % scanned, + scanned > 20) + + CHECKS = [v for k, v in sorted(globals().items()) if k.startswith("check_")]