Skip to content

Mirror engines survive a block with no pool, solves stay within budget, canary is red unless it tested Etsy #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jehrr merged 1 commit into from
Oct 15, 2026
Merged

Mirror engines survive a block with no pool, solves stay within budget, canary is red unless it tested Etsy #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 56 additions & 9 deletions .github/ci_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager")

CLIS = ["playwright_scraper.py", "puppeteer_scraper.py", "selenium_scraper.py",
"scraper_api_client.py", "fingerprint_client.py", "env_config.py"]
"scraper_api_client.py", "fingerprint_client.py", "env_config.py",
"diff_runs.py"]

SAMPLE_FILES = ("sample_output.json", "sample_output.csv")

Expand Down Expand Up @@ -85,7 +86,7 @@
# into pip's vendored code and flags a 32-hex string in `_elffile.py` as
# key-shaped. Correct about the string, wrong about the file — and it is the
# FIRST thing a new user sees from `python3 smoke_test.py`. A guard people
# have to argue with is one they learn to suppress (CLAUDE.md §22).
# have to argue with is one they learn to suppress.
#
# Deliberately NOT a narrowing of what gets scanned: an untracked file is
# still read, because a key pasted into a scratch file beside the scripts is
Expand All @@ -112,18 +113,55 @@ def _is_in_virtualenv(path):
return False


# Read too, though nothing here is source: data and dumps are where a key or
# a session arrives by accident (a fixture captured with a key in a request,
# a `--dump-html` of a page that echoed one), and `.env.bak`/`.env.local`
# are what a working `.env` gets renamed to. These shapes are git-ignored by
# design, so a hit in an IGNORED one is reported as a warning — the ignore
# rule is what keeps it out of a commit, and a local key in a local file is
# not a defect. A hit in one git would NOT ignore fails like any other.
DATA_SUFFIXES = (".json", ".html", ".csv", ".toml", ".cfg", ".ini")
DATA_NAMES = ("Dockerfile",)

# Skipped by name, and only RELATIVE to the repository: tested against the
# path's absolute parts, a clone that merely lives under a directory called
# `venv` (or `.git`) excluded every file, and the scan reported "0 files
# scanned, nothing credential-shaped" — a clean pass over nothing.
SKIPPED_DIRS = {".git", "__pycache__", ".venv", "venv", ".pytest_cache"}


def _is_data_file(path):
name = path.name
return (path.suffix in DATA_SUFFIXES or name in DATA_NAMES
or (name.startswith(".env.") and name != ".env.example"))


def scanned_files():
for path in sorted(REPO.rglob("*")):
if not path.is_file() or path.suffix not in SCANNED_SUFFIXES:
if not path.is_file():
continue
if any(part in {".git", "__pycache__", ".venv", "venv"}
for part in path.parts):
if path.suffix not in SCANNED_SUFFIXES and not _is_data_file(path):
continue
if any(part in SKIPPED_DIRS for part in path.relative_to(REPO).parts):
continue
if _is_in_virtualenv(path):
continue
yield path


def _git_ignored(paths):
"""The subset of `paths` git would ignore; empty outside a checkout."""
if not paths:
return set()
try:
done = subprocess.run(
["git", "check-ignore", "--stdin"], cwd=REPO, capture_output=True,
text=True, input="\n".join(str(p.relative_to(REPO)) for p in paths))
except FileNotFoundError:
return set()
return {REPO / line for line in done.stdout.splitlines() if line}


def help_check():
failed = []
for name in CLIS:
Expand Down Expand Up @@ -190,25 +228,34 @@ def sample_check():


def secret_check():
failed = []
failed, warned = [], []
scanned = 0
for path in scanned_files():
files = list(scanned_files())
ignored_data = _git_ignored([p for p in files if _is_data_file(p)])
for path in files:
scanned += 1
sink = warned if path in ignored_data else failed
for lineno, line in enumerate(
path.read_text(encoding="utf-8", errors="replace").splitlines(), 1):
rel = path.relative_to(REPO)

if CREDENTIALLED_URL.search(line) and not any(
token in line for token in CREDENTIAL_ALLOWED):
failed.append(f"{rel}:{lineno} looks like a URL with real "
sink.append(f"{rel}:{lineno} looks like a URL with real "
f"credentials in it")

for match in HEX32.findall(line):
if any(token in line.lower() for token in HEX32_ALLOWED):
continue
failed.append(f"{rel}:{lineno} contains {match[:6]}… — a "
sink.append(f"{rel}:{lineno} contains {match[:6]}… — a "
f"32-char hex string, the shape of a 2captcha key")

for line in warned:
print(f"WARNING {line} (git-ignored, so not a commit risk while the "
f"ignore rule holds; delete it if it is not yours)")
if scanned == 0:
failed.append("0 files scanned — a scan that read nothing cannot say "
"the repository is clean")
if not failed:
print(f"ok {scanned} files scanned, nothing credential-shaped")
return failed
Expand Down
113 changes: 37 additions & 76 deletions .github/workflows/canary.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ name: canary
# why. That leaves a real gap: a markup change on the site itself only shows
# up when a user hits it and files an issue, while the offline badge stays
# green throughout. This workflow is the other half — a real run against a
# real search URL — so a site-side break is caught here first. It runs ON
# DEMAND rather than on a schedule, for the reason set out below.
# real search URL — so a site-side break is caught here first. It runs
# weekly and on demand.
#
# THIS CANARY CANNOT RUN FROM A BARE RUNNER, AND THAT IS NOT PESSIMISM.
# ------------------------------------------------------------------------
Expand All @@ -22,10 +22,10 @@ name: canary
# browser here. So this canary gates on ETSY_CDP_ENDPOINT — the Scraping
# Browser API — rather than on a proxy secret like its sibling repos do.
#
# The live steps are SKIPPED, not failed, when the secret is absent. The job
# goes green with a notice saying why. A check that is always red teaches
# everyone to ignore checks, which is worse than having no check at all. Set
# the secret and the same workflow starts doing real work with no edit:
# WITHOUT THAT SECRET THIS JOB FAILS, with an error naming it. It used to
# skip and go green, and a green badge over a run that never reached the site
# reads as "the parser still works" when nothing was tested. Red until the
# secret is set is the honest state:
#
# gh secret set ETSY_CDP_ENDPOINT --repo 2scraper/etsy-scraper
#
Expand All @@ -40,27 +40,14 @@ name: canary
# its own CI; and an unset secret arrives as an empty string, which the
# loader already treats as "not configured", so there is no commented-out
# line for someone to forget to uncomment.
# ON DEMAND, NOT ON A SCHEDULE — and that is forced by the credential rather
# than chosen.
#
# A Scraping Browser endpoint on this account does not survive a day. A daily
# cron against a credential with that lifetime produces one of two useless
# outcomes: a red badge every day (if a dead endpoint fails the job), or a
# green badge every day that has tested nothing (if it does not). The second
# is the more dangerous, because green reads as "the parser still works".
#
# So the schedule is off. Dispatch it by hand, with a fresh secret, when the
# answer matters — before a release, or when someone reports the parser
# missing a column. That is honest about what the check covers: the offline
# suite runs on every push and pins the parser against real captures; this
# workflow is the only thing that can catch ETSY changing its markup, and it
# runs when someone asks it to.
#
# Put the schedule back the day a long-lived credential exists:
# schedule:
# - cron: "23 6 * * *"
# WEEKLY. How long a Scraping Browser endpoint stays valid has not been
# measured for this account, so the schedule is not built around a lifetime
# guess: if the secret stops authenticating, the run goes red and says so,
# and that is the signal to refresh it.
on:
workflow_dispatch:
schedule:
- cron: "23 6 * * 1"

jobs:
live-run:
Expand All @@ -72,20 +59,18 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Explain a skipped run
- name: Fail when the live run cannot happen
if: env.HAVE_CDP != 'true'
run: |
echo "::notice::Skipping the live run: no ETSY_CDP_ENDPOINT secret is set."
echo "::error::The ETSY_CDP_ENDPOINT secret is not set, so this canary cannot reach etsy.com and has tested nothing."
echo "Etsy's DataDome refuses datacentre addresses, and a GitHub runner"
echo "is one — a residential proxy alone was measured as not enough for"
echo "an automated browser either. A run from here would be red every"
echo "day whatever the state of the code. Set a Scraping Browser API"
echo "endpoint (with a pid reserved for CI) to turn this canary on:"
echo "is one; a residential proxy alone was measured as not enough for"
echo "an automated browser either. Set a Scraping Browser API endpoint"
echo "(with a pid reserved for CI):"
echo
echo " gh secret set ETSY_CDP_ENDPOINT --repo 2scraper/etsy-scraper"
echo
echo "The offline suite (tests.yml) covers everything that does not"
echo "need the network, and it runs on every push."
echo "### Canary did NOT test the site: ETSY_CDP_ENDPOINT is not set" >> "$GITHUB_STEP_SUMMARY"
exit 1

- uses: actions/setup-python@v5
if: env.HAVE_CDP == 'true'
Expand Down Expand Up @@ -136,58 +121,34 @@ jobs:
- name: Interpret the exit code
id: verdict
if: env.HAVE_CDP == 'true'
# ACCESS CONDITIONS ARE NOT DEFECTS, and on this site that
# distinction decides whether the badge means anything.
#
# An earlier version failed the job on ANY non-zero exit. That makes
# the canary red for two things that are nobody's bug:
#
# * the SECRET EXPIRED. A Scraping Browser credential is not
# forever; when it stops authenticating the run exits 5, and a
# daily cron then paints the badge red every day until someone
# refreshes it. A check that is always red teaches everyone to
# ignore checks, which is worse than having no check.
# * ACCESS IS INTERMITTENT PER PROFILE. Measured 2026-09-09/10 on
# one account: a fresh profile served three full pages in the
# morning, and that evening two fresh profiles were refused with
# t=bv through all their retries while a third served a shop
# front in full, twice. Same code, same zone, same URLs.
#
# So: 3 (blocked), 5 (remote API error) and 6 (partial) are reported
# as WARNINGS and do not fail the job. What stays red is a run that
# got in and found the wrong thing, or one that broke:
#
# 1 crashed
# 2 the workflow's own arguments are wrong
# 4 served a page and parsed ZERO rows from a search that returns
# tens of thousands — the tile anchor moved, which is exactly
# what this canary exists to catch
#
# A warning must not read as a pass, either. Each one writes to the
# step summary as well, so the run page says in words that the canary
# did not actually test anything this time.
# ANY RUN THAT DID NOT TEST THE SITE IS RED. Only exit 0 goes on to
# the data checks below. A block (3), a run that never obtained the
# page or whose parser read nothing from a served page (5), a partial
# run (6) and zero rows (4) all fail the job: each means this
# workflow's one purpose — showing that a live Etsy search still
# parses — was not achieved, whatever the reason. The message says
# which reason, so an access problem is not mistaken for a parser
# regression.
run: |
set -e
code="${{ steps.run.outputs.exit_code }}"
tested=true
tested=false
case "$code" in
0)
tested=true
echo "OK — listings parsed from a live run." ;;
3)
tested=false
echo "::warning::Blocked before parsing (exit 3) — an ACCESS condition, not a code defect, so this run is not failed. Read the log for the DataDome 't' value: t=bv means the profile or its exit is banned and no captcha solve can help; t=fe means a solvable challenge appeared and no key was configured. Try a different --cdp-endpoint pid."
echo "### Canary did NOT test anything (exit 3, blocked)" >> "$GITHUB_STEP_SUMMARY"
echo "Etsy refused the request before any markup was parsed. The parser was not exercised. This is access, not a regression — but it also means the site-side check did not happen." >> "$GITHUB_STEP_SUMMARY" ;;
echo "::error::Blocked before parsing (exit 3): the site was not tested. Read the log for the DataDome 't' value: t=bv means the profile or its exit is refused and no solve helps; t=fe is a solvable challenge with no key configured. Try a different pid in ETSY_CDP_ENDPOINT."
echo "### Canary did NOT test the site (exit 3, blocked)" >> "$GITHUB_STEP_SUMMARY"
exit 1 ;;
5)
tested=false
echo "::warning::Remote API error (exit 5) — the Scraping Browser endpoint refused the connection, so nothing was tested and this run is not failed. If the log says 'profile_locked', another run still holds this pid: give CI its own. Otherwise the ETSY_CDP_ENDPOINT secret has most likely expired and needs refreshing."
echo "### Canary did NOT test anything (exit 5, endpoint refused)" >> "$GITHUB_STEP_SUMMARY"
echo "The most likely cause is an expired \`ETSY_CDP_ENDPOINT\` secret. Refresh it, or remove it to go back to the explicit SKIP path." >> "$GITHUB_STEP_SUMMARY" ;;
echo "::error::Nothing was obtained (exit 5). The log names the cause: the endpoint refused the connection (401: check or refresh ETSY_CDP_ENDPOINT; profile_locked: another run holds this pid), a navigation that never completed, or parser_found_nothing — a served page that states listings and parsed to none, which means the tile markup moved. See stop_reason in the log and the HTML dump."
echo "### Canary did NOT test the site (exit 5)" >> "$GITHUB_STEP_SUMMARY"
exit 1 ;;
6)
tested=false
echo "::warning::Partial run (exit 6) — some pages were gathered and the run then stopped early, usually a mid-run block. canary_run.meta.json names which pages failed. Not failed, because on this site that is an access condition; if it repeats on every run, suspect pagination instead."
echo "::error::Partial run (exit 6): some pages were gathered and the run then stopped early. canary_run.meta.json names which pages failed and why (stop_reason)."
echo "### Canary tested only PART of the run (exit 6)" >> "$GITHUB_STEP_SUMMARY"
echo "See \`pages_failed\` in the sidecar artefact." >> "$GITHUB_STEP_SUMMARY" ;;
exit 1 ;;
4)
echo "::error::Ran fine but parsed 0 rows (exit 4) on a search that returns tens of thousands of listings. The tile anchor (data-listing-id) has probably moved — this is the regression this canary exists to catch. The HTML dump in the artefacts tells you."
exit 1 ;;
Expand Down
Loading
Loading