diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index e4ee8a2..d3e7363 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -274,3 +274,223 @@ jobs: canary_*_debug.png if-no-files-found: ignore retention-days: 14 + + # THE LIVE CROSS-ENGINE CHECK. + # + # The offline suite already asserts that the three engines take the same + # flags, bind the same shared-module signatures and share one `finish_run`, + # so their exit codes and sidecars cannot drift. What none of that tests is + # whether they still agree about the SITE — and only the primary engine was + # ever run against it on a schedule. A third-party audit raised exactly + # this, and offered two remedies: declare Playwright the only supported + # engine, or run a small live check on each. The README promises three, so + # this is the second. + # + # Compared on IDS AND SCHEMA, not on values. The three runs happen minutes + # apart and a prediction market reprices continuously, so requiring equal + # prices would produce a red badge about nothing — and a check that is + # always red teaches everyone to ignore checks. What must not differ is + # which markets were found, what the columns are called, and the fields a + # market does not change by trading. + cross-engine: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # One venv per engine, from that engine's own requirements file. The + # three declare mutually unsatisfiable pins (pyee <12 vs >=13, urllib3 + # <2.0 vs >=2.6), so a single environment would have pip resolve the + # conflict by reaching for whatever it can — which in a sibling repo + # silently installed a stub version and left CI green against a release + # the repo does not support. + - name: Install each engine in its own virtualenv + run: | + for engine in playwright selenium puppeteer; do + python -m venv ".venv-$engine" + ".venv-$engine/bin/pip" install --quiet --upgrade pip + ".venv-$engine/bin/pip" install --quiet \ + -r requirements.txt -r "requirements-$engine.txt" + done + .venv-playwright/bin/playwright install --with-deps chromium + + # ONE event page, not a listing. An event page is a single fetch whose + # row set is decided by the event rather than by what the front page is + # promoting this minute, so the three runs are comparable without + # pinning a URL that will resolve and 404 one day: the URL comes from + # the listing, chosen at run time. + - name: Pick an event the listing names right now + id: pick + shell: bash + run: | + set +e + .venv-playwright/bin/python playwright_scraper.py \ + --url "https://polymarket.com/predictions" \ + --mode markets --format json --out cross_listing + code=$? + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + if [ "$code" != "0" ]; then exit 0; fi + url=$(python -c " + import json + rows = json.load(open('cross_listing.json')) + for r in rows: + if r.get('event_url'): + print(r['event_url']); break + ") + echo "url=$url" >> "$GITHUB_OUTPUT" + echo "picked $url" + + - name: Run the same event page through all three engines + if: steps.pick.outputs.exit_code == '0' + shell: bash + run: | + # Spelled out rather than looped over `$engine`. A loop reads + # well and makes the engine names invisible to anything grepping + # this file — including the offline check that asserts the canary + # still covers every engine the README promises. Three lines are + # worth a check that can fail. + # + # `set +e`: a refused engine must not stop its twins, and the + # comparison step decides what a missing run means. + set +e + .venv-playwright/bin/python playwright_scraper.py \ + --url "${{ steps.pick.outputs.url }}" \ + --mode event --format json --out cross_playwright + echo "playwright_scraper.py exited $?" + .venv-selenium/bin/python selenium_scraper.py \ + --url "${{ steps.pick.outputs.url }}" \ + --mode event --format json --out cross_selenium + echo "selenium_scraper.py exited $?" + .venv-puppeteer/bin/python puppeteer_scraper.py \ + --url "${{ steps.pick.outputs.url }}" \ + --mode event --format json --out cross_puppeteer + echo "puppeteer_scraper.py exited $?" + exit 0 + + - name: The three must agree on ids and schema + if: steps.pick.outputs.exit_code == '0' + shell: python + run: | + import json + from pathlib import Path + + ENGINES = ("playwright", "selenium", "puppeteer") + + # An ALLOWLIST of fields a market does not change by trading, not a + # blocklist of the ones it does. Inverted deliberately: with a + # blocklist a column added later is compared by default and starts + # failing the badge the first time the market moves, which is how a + # live check teaches people to ignore it. Schema drift is caught by + # the column-set comparison below regardless of this list. + # + # `data_source` earns its place: if one engine fell back to the DOM + # while its twins read the payload, every one of these would still + # match and only this column would say so. + STABLE = { + "source", "url", "sku", "title", "outcomes", "group_item_title", + "event_id", "event_slug", "event_title", "event_url", + "category", "subcategory", "tags", "start_date", "end_date", + "market_id", "condition_id", "clob_token_ids", "neg_risk", + "data_source", "page", "position", + } + + runs, missing = {}, [] + for engine in ENGINES: + path = Path("cross_%s.json" % engine) + if not path.exists(): + missing.append(engine) + continue + runs[engine] = json.loads(path.read_text()) + + # A run that was refused says something about this runner's + # address, not about the engine. Warn in words rather than failing + # the badge — but never let the warning read as a pass. + if missing: + summary = ("%s produced no rows; the cross-engine comparison " + "did NOT run for %s today." + % (", ".join(missing), ", ".join(missing))) + print("::warning::" + summary) + with open("%s" % __import__("os").environ["GITHUB_STEP_SUMMARY"], + "a") as fh: + fh.write("### cross-engine canary\n\n" + summary + "\n") + if len(runs) < 2: + print("fewer than two engines returned rows — nothing to " + "compare. Not a pass.") + raise SystemExit(0) + + fail = [] + base = ENGINES[0] if ENGINES[0] in runs else sorted(runs)[0] + base_rows = {r["sku"]: r for r in runs[base]} + + for engine, rows in runs.items(): + if engine == base: + continue + other = {r["sku"]: r for r in rows} + + # IDS. The same event page read twice must name the same + # markets, whichever driver fetched it. + if set(other) != set(base_rows): + only_base = sorted(set(base_rows) - set(other))[:3] + only_other = sorted(set(other) - set(base_rows))[:3] + fail.append("%s and %s disagree on which markets exist " + "(%s only in %s, %s only in %s)" + % (base, engine, only_base, base, + only_other, engine)) + + # SCHEMA. A column that exists for one engine and not another + # is the drift this check is for. + for sku in sorted(set(other) & set(base_rows)): + a, b = base_rows[sku], other[sku] + if set(a) != set(b): + fail.append("%s and %s produce different columns: %s" + % (base, engine, + sorted(set(a) ^ set(b)))) + break + + # The STABLE fields of a market the site did not change. + for sku in sorted(set(other) & set(base_rows)): + a, b = base_rows[sku], other[sku] + differing = [k for k in sorted(STABLE & set(a)) + if a[k] != b.get(k)] + if differing: + fail.append("%s and %s disagree on %s for %s: %r vs %r" + % (base, engine, differing[0], sku, + a[differing[0]], + b.get(differing[0]))) + break + + # The allowlist has to name real columns, or it silently shrinks to + # nothing and this step passes by comparing almost no fields. + columns = set(next(iter(runs.values()))[0]) if any( + runs.values()) else set() + unknown = sorted(STABLE - columns) + if columns and unknown: + fail.append("STABLE names %s, which are not columns — the " + "comparison was narrower than it reads" % unknown) + if columns and len(STABLE & columns) < 15: + fail.append("only %d stable columns compared; the allowlist has " + "drifted away from the schema" + % len(STABLE & columns)) + + print("compared %s: %s" + % (sorted(runs), {e: len(r) for e, r in runs.items()})) + if fail: + for f in fail: + print("FAIL: " + f) + raise SystemExit(1) + print("the engines agree on ids, columns and every stable field.") + + - name: Upload what the three engines produced + if: always() + uses: actions/upload-artifact@v4 + with: + name: cross-engine-artifacts + path: | + cross_*.json + cross_*.meta.json + if-no-files-found: ignore diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f229e1..60cb0d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,65 @@ from a row count, it leads the section in a blockquote. ## [Unreleased] +### Added + +- **The canary now runs all three engines against the live site, daily.** + The audit's third finding, and a fair one: the offline suite asserts the + engines take the same flags, bind the same shared-module signatures and + share one `finish_run` — so their exit codes and sidecars cannot drift — + but none of that tests whether they still agree about the SITE, and only + the primary engine ran against it on a schedule. + + The audit offered two remedies and this takes the second. The first was to + declare Playwright the only supported engine and demote the others, which + would contradict the README and throw away three working engines to close + a gap that one job closes. + + Compared on **ids and schema, not on values**: the three runs are minutes + apart and a prediction market reprices continuously, so requiring equal + prices would produce a red badge about nothing. What must not differ is + which markets were found, what the columns are called, and the fields a + market does not change by trading — an explicit allowlist, inverted from + the obvious blocklist so a column added later is not compared by default + and does not start failing the badge the first time the market moves. The + allowlist is itself checked against the real columns, or it could shrink + to nothing and the step would pass by comparing almost nothing. + + `data_source` is in that allowlist on purpose: if one engine fell back to + the DOM while its twins read the payload, every other field would still + match and only that column would say so. + + Verified by running it here before shipping it: all three engines on one + event page returned the same five markets, the same 40 columns and the + same 22 stable fields, with `scraped_at` the only field that differed. + +- **The Docker base image is pinned by digest**, with the tag kept beside it + so a reader can see which release the digest is. `python:3.12-slim` is a + moving target, so the same Dockerfile built a month apart is a different + image: a release and its own rebuild are not the same artefact, and a + regression arriving through the base reads as a regression in this code. + Built and run after pinning — 53 markets live from inside the container. + +### Deliberately not done + +- **Lock files for the Python dependencies**, which the audit recommends in + the same breath as the Docker pin. The two are not the same trade. A lock + file would stop the daily canary from testing new releases of `requests`, + `beautifulsoup4` and the engine libraries — and that daily test is the + thing that catches a dependency change the morning it happens rather than + whenever someone next bumps a pin. The image is a build artefact and + should be reproducible; the canary is a detector and should be exposed. + Lower bounds plus a daily live run is the combination that reports + breakage, and pinning both halves would buy reproducibility by turning the + detector off. + +- **A run identifier across the JSON, CSV and sidecar set.** The failure it + guards against is already impossible by construction: a failed run writes + no sidecar at all, `save` refuses to overwrite good output with an empty + result, and as of this release each file is replaced by an atomic rename. + A transaction id would add a second thing to keep consistent, which is a + second thing that can disagree. + ### Fixed - **An interrupted write destroyed the previous good run's output.** The diff --git a/Dockerfile b/Dockerfile index bb0f63e..2c25e96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,25 @@ # Pass --proxy/--twocaptcha-key the same way as running locally, or mount a # .env at /app/.env — nothing here bakes in a credential, and .dockerignore # keeps one out of the build context. -FROM python:3.12-slim +# PINNED BY DIGEST, and the tag kept beside it so a reader can see which +# release this is. `python:3.12-slim` is a moving target: the same Dockerfile +# built a month apart is a different image, which is exactly the +# reproducibility gap an audit raised — a release built today and the same +# release rebuilt later are not the same artefact, and a regression that +# arrives through the base image looks like a regression in this code. +# +# Resolved 2026-10-09 from the tag it replaces, so this IS 3.12-slim as of +# that date rather than a version chosen here. To move it deliberately: +# +# docker pull python:3.12-slim +# docker inspect --format='{{index .RepoDigests 0}}' python:3.12-slim +# +# This is the half of "pin everything" worth taking. Lock files for the +# Python dependencies are deliberately NOT added — see CHANGELOG — because +# they would stop the daily canary from testing new releases of those +# libraries, which is the thing that catches a dependency change the morning +# it happens. +FROM python:3.12-slim@sha256:05cda9777409a9c3ffddd94a4c476b79f0769a0b4857f0c7ed9226b6800b0d6f WORKDIR /app diff --git a/README.md b/README.md index feb1b83..5d077f0 100644 --- a/README.md +++ b/README.md @@ -422,6 +422,17 @@ one real `--mode events` walk and one listing run daily, and asserts the pagination, the deep columns, the price coverage and the site's own `total_events`. +It also runs **all three engines against one event page every day** and +checks that they still agree. The offline suite already holds them to the +same flags, the same shared-module signatures and one `finish_run`, so their +exit codes and sidecars cannot drift — but only a live run can show whether +they still read the SITE the same way. The comparison is on ids and schema, +never on prices: the three runs are minutes apart and a prediction market +reprices continuously, so what must match is which markets were found, what +the columns are called, and every field a market does not change by trading. +Measured 2026-10-09 on one event page: the same five markets, the same forty +columns, and `scraped_at` the only field that differed between engines. + --- ## Licence diff --git a/smoke_test.py b/smoke_test.py index 6fab859..0749c5e 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2241,6 +2241,45 @@ def test_virtualenvs_in_the_tree_are_ignored(): return ok +def test_the_canary_covers_all_three_engines(): + group("The live canary runs every engine the README promises") + ok = True + # The offline suite asserts the three engines take the same flags, bind + # the same shared-module signatures and share one `finish_run`, so their + # exit codes and sidecars cannot drift. None of that tests whether they + # still agree about the SITE, and for a while only the primary engine ran + # against it on a schedule. + # + # A third-party audit offered two remedies: declare Playwright the only + # supported engine, or check the others live. The README promises three, + # so this pins the second — and pins it against the README rather than + # against a hardcoded list, so dropping an engine from one without the + # other fails here. + path = os.path.join(REPO_ROOT, ".github", "workflows", "canary.yml") + if not os.path.exists(path): + # The Dockerfile copies no .github/, and this suite runs inside the + # image at build time. Keyed on the whole directory being absent, not + # on this file being missing, so a deleted workflow still fails. + if not os.path.exists(os.path.join(REPO_ROOT, ".github")): + print(" SKIP no .github/ here (the image carries none)") + return ok + return check("the canary workflow exists", False) + canary = open(path, encoding="utf-8").read() + for engine in ENGINES: + # INVOKED, not merely mentioned. The first version of this check + # looked for the file name anywhere in the workflow, and an `echo` + # naming the engine was enough to satisfy it — planting the fault + # (removing the line that runs selenium, leaving the line that logs + # its exit code) left the suite green. + ok &= check("the canary runs %s" % engine, + bool(re.search(r"bin/python\s+%s\.py" % re.escape(engine), + canary))) + ok &= check("...and compares them on ids rather than on prices", + "STABLE" in canary and "disagree on which markets exist" + in canary) + return ok + + def test_tree_scanners_skip_virtualenvs(): group("Every tree-wide scanner skips a virtualenv (structurally)") ok = True @@ -2415,6 +2454,26 @@ def test_dockerfile_matches_its_entrypoint(): if not os.path.exists(path): return check("a Dockerfile exists", False) dockerfile = open(path, encoding="utf-8").read() + + # The base image is pinned BY DIGEST. `python:3.12-slim` is a moving + # target, so the same Dockerfile built a month apart is a different + # image: a release and its own rebuild are then not the same artefact, + # and a regression arriving through the base reads as a regression in + # this code. Raised by a third-party audit as part of a wider "pin + # everything"; this is the half that was taken, and the CHANGELOG says + # why lock files were not. + froms = [l.strip() for l in dockerfile.splitlines() + if l.strip().upper().startswith("FROM ")] + ok &= check("the Dockerfile has exactly one FROM (found %d)" % len(froms), + len(froms) == 1) + for line in froms: + ok &= check("...pinned by digest, not by a moving tag (%s)" + % line[:60], + bool(re.search(r"@sha256:[0-9a-f]{64}\b", line))) + ok &= check("...with the tag kept beside it, so a reader can see " + "which release that digest is", + bool(re.search(r"FROM\s+\S+:[\w.\-]+@sha256:", line))) + # Join backslash continuations first: the COPY list spans five lines, and # a line-by-line reader sees an empty list and passes vacuously. joined = re.sub(r"\\\s*\n\s*", " ", dockerfile) @@ -3002,6 +3061,7 @@ def main() -> int: ok &= test_engine_parity(skips) ok &= test_module_attributes_exist(skips) ok &= test_virtualenvs_in_the_tree_are_ignored() + ok &= test_the_canary_covers_all_three_engines() ok &= test_tree_scanners_skip_virtualenvs() ok &= test_no_dead_public_names() ok &= test_no_undefined_names()