From c4816f08c93c1d91b948a463a2ad9471013fc581 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Tue, 1 Sep 2026 10:27:24 +1000 Subject: [PATCH 1/2] Phase 5: the weekly canary and refresh-as-PR for dynamic snapshots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One manifest-driven workflow, .github/workflows/refresh-snapshots.yml: - canary: every `class: dynamic-snapshot` builder runs with --out-dir (fetch + validate, no commit). A failure opens or updates one `upstream-break` issue, classified by exit code — 2 is a ValidationError (the data broke the contract; a human), anything else is the fetch (a retry). Consumers read the last-good snapshot either way. - refresh: datasets that are due — cadence elapsed since `retrieved`, `integrity.upstream.status: diverged`, or never refreshed — get the builder run in place, the manifest stamped, CATALOG.md regenerated and a PR on refresh/ whose body is the builder's overlap summary. A later run updates the same PR. Opened with QUANTECON_SERVICES_PAT when the org secret reaches this repo, else the workflow token (documented cost: the required check will not self-start on such a PR). scripts/snapshots.py is the manifest side: `list`, `due`, `stamp` (edits the manifest TEXT so the comments survive, then re-parses to prove the stamp reads back), `pr-body`. The builder contract gains --summary-json, a ValidationError with exit code 2, and the run summary from validate(); builders/_template.py is the copy-able skeleton #14 asked for. business_cycle_data.csv.yml is made stampable: its title no longer embeds the end year, and every stamped field is a single-line value with its reasoning in comments above it. Docs: PLAN Phase 5 boxes (refresh, canary ticked; fan-out policy noted), AGENTS.md builder contract and repo map, builders/README.md, and README.md's layout table, which still said builders lived in scripts/. Tested locally end to end: due → builder --summary-json → stamp → the manifest reads back verified, check_consumed_files clean, due flips to skip, pr-body renders; the stamped state was then reverted so this PR changes no published bytes. See #14. Co-Authored-By: Claude Fable 5 --- .github/workflows/refresh-snapshots.yml | 265 +++++++++++++++++++ AGENTS.md | 6 +- CATALOG.md | 2 +- PLAN.md | 6 +- README.md | 25 +- builders/README.md | 15 +- builders/_template.py | 122 +++++++++ builders/business_cycle.py | 101 ++++++-- lectures/business_cycle_data.csv.yml | 38 +-- scripts/snapshots.py | 329 ++++++++++++++++++++++++ 10 files changed, 857 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/refresh-snapshots.yml create mode 100644 builders/_template.py create mode 100644 scripts/snapshots.py diff --git a/.github/workflows/refresh-snapshots.yml b/.github/workflows/refresh-snapshots.yml new file mode 100644 index 0000000..63a0361 --- /dev/null +++ b/.github/workflows/refresh-snapshots.yml @@ -0,0 +1,265 @@ +# Dynamic snapshots (PLAN Phase 5): the weekly sources-alive canary and the +# refresh-as-PR, in one manifest-driven workflow. +# +# canary every `class: dynamic-snapshot` dataset with a runnable builder: +# fetch + validate to a scratch directory, no commit. A failure +# opens (or updates) one `upstream-break` issue here, classified +# by the builder's exit code — 2 is a ValidationError (the data +# broke the contract; a human), anything else is a fetch failure +# (retry). Consumers are unaffected either way: they read the +# last-good snapshot. This is where live-API fragility lives now, +# instead of in the lecture repos' CI. +# refresh the datasets that are DUE — cadence elapsed since `retrieved`, +# `integrity.upstream.status: diverged`, or never refreshed — get +# the builder run in place, the manifest stamped +# (scripts/snapshots.py), CATALOG.md regenerated, and a PR on +# `refresh/`. A later run updates the same branch and PR. +# Nothing lands on main without a review; the PR body carries the +# builder's overlap summary, which is the review surface. +# +# Who gets told, per AGENTS.md "Refresh, break, or schema change": a break is +# an issue HERE; a merged refresh reaches consumers per the manifest's +# `on_refresh` (the fan-out is not wired yet — the PR body lists what it would +# do, and today no snapshot has a consumer). +# +# Token: the PR is opened with QUANTECON_SERVICES_PAT when the org secret is +# available to this repo, falling back to the workflow token. The fallback +# works, with one known cost — GitHub does not run `pull_request` workflows +# for PRs opened by GITHUB_TOKEN, so the required `consumed-files` check will +# not start on its own; close and reopen the PR (or push to it) to trigger it. + +name: refresh-snapshots + +on: + schedule: + - cron: "17 6 * * 1" # weekly, an hour after audit-dashboard + workflow_dispatch: + inputs: + dataset: + description: "one dataset (its lectures/ filename), or blank for all" + required: false + default: "" + force: + description: "refresh even if not due" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: refresh-snapshots + cancel-in-progress: false + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + all: ${{ steps.plan.outputs.all }} + due: ${{ steps.plan.outputs.due }} + steps: + - uses: actions/checkout@v4 + with: + lfs: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pyyaml + - id: plan + env: + DATASET: ${{ inputs.dataset }} + FORCE: ${{ inputs.force }} + run: | + flags="" + [ -n "$DATASET" ] && flags="$flags --dataset $DATASET" + [ "$FORCE" = "true" ] && flags="$flags --all" + all=$(python scripts/snapshots.py list | jq -c .) + due=$(python scripts/snapshots.py due $flags | jq -c .) + echo "all=$all" >> "$GITHUB_OUTPUT" + echo "due=$due" >> "$GITHUB_OUTPUT" + echo "canary: $(echo "$all" | jq -r '.[].dataset' | tr '\n' ' ')" + echo "due: $(echo "$due" | jq -r '.[].dataset' | tr '\n' ' ')" + + canary: + needs: plan + if: needs.plan.outputs.all != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + snapshot: ${{ fromJson(needs.plan.outputs.all) }} + name: canary (${{ matrix.snapshot.dataset }}) + # A matrix cannot expose one output per leg, so a failing leg leaves its + # failure.env + log as an artifact and the notify job reads those. + steps: + - uses: actions/checkout@v4 + with: + lfs: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -r requirements.txt + - name: Fetch + validate, no commit + id: run + env: + BUILDER: ${{ matrix.snapshot.builder }} + DATASET: ${{ matrix.snapshot.dataset }} + run: | + set +e + python "$BUILDER" --out-dir canary-out --summary-json summary.json 2>&1 | tee builder.log + code=${PIPESTATUS[0]} + set -e + case "$code" in + 0) kind=ok ;; + 2) kind=validation ;; + *) kind=fetch ;; + esac + echo "kind=$kind" >> "$GITHUB_OUTPUT" + { + echo "dataset=$DATASET" + echo "builder=$BUILDER" + echo "kind=$kind" + echo "code=$code" + } > failure.env + exit "$code" + - name: Keep the failure for the notifier + if: failure() + uses: actions/upload-artifact@v4 + with: + name: canary-failure-${{ matrix.snapshot.stem }} + path: | + failure.env + builder.log + retention-days: 7 + + notify: + if: failure() + needs: [plan, canary] + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: canary-failure-* + path: failures + - name: Open (or update) the upstream-break issue + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + gh label create upstream-break --force --color b60205 \ + --description "A dynamic snapshot's builder failed against its live source" + + { + echo "The weekly sources-alive canary failed. Nothing was written: every consumer keeps reading the last-good snapshot, so no lecture is affected — this is the alarm moving from the lecture repos' CI to here, doing its job." + echo + for dir in failures/*/; do + [ -f "$dir/failure.env" ] || continue + . "$dir/failure.env" + case "$kind" in + validation) what="**validation failed** (exit 2) — the fetched data broke the published contract. A human decides: absorb the upstream change in the builder's \`pre_process\` stage so the published schema is unchanged, or, if it cannot honestly be absorbed, plan a new-filename vintage (AGENTS.md, \"Refresh, break, or schema change\")." ;; + *) what="**fetch failed** (exit $code) — the upstream or the network, not the data. Re-run the workflow; if it fails again, the source has moved or gone." ;; + esac + echo "### \`$dataset\` — $what" + echo + echo "Builder: \`$builder\`. Last lines of its log:" + echo + echo '```' + tail -n 15 "$dir/builder.log" + echo '```' + echo + done + echo "Run: $RUN_URL" + echo + echo "_Posted automatically. Later failures comment here rather than opening new issues, so close this once the canary is green._" + } > body.md + + open=$(gh issue list --label upstream-break --state open --limit 1 \ + --json number --jq '.[0].number // empty') + if [ -n "$open" ]; then + gh issue comment "$open" --body-file body.md + else + gh issue create --title "refresh-snapshots: a dynamic snapshot's builder is failing" \ + --label upstream-break --assignee mmcky --body-file body.md + fi + + refresh: + needs: [plan, canary] + if: needs.plan.outputs.due != '[]' && needs.canary.result == 'success' + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + strategy: + fail-fast: false + max-parallel: 1 + matrix: + snapshot: ${{ fromJson(needs.plan.outputs.due) }} + name: refresh (${{ matrix.snapshot.dataset }}) + steps: + - uses: actions/checkout@v4 + with: + lfs: false + # The PAT, when present, is what lets the PR trigger `consumed-files` + # (see the header). `persist-credentials` keeps it for the push. + token: ${{ secrets.QUANTECON_SERVICES_PAT || github.token }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -r requirements.txt + - name: Run the builder in place and stamp the manifest + env: + BUILDER: ${{ matrix.snapshot.builder }} + DATASET: ${{ matrix.snapshot.dataset }} + run: | + python "$BUILDER" --summary-json summary.json + if git diff --quiet -- "lectures/$DATASET"; then + echo "bytes_changed=false" >> "$GITHUB_ENV" + else + echo "bytes_changed=true" >> "$GITHUB_ENV" + fi + python scripts/snapshots.py stamp "$DATASET" --summary summary.json + python scripts/build_catalog.py + python .github/scripts/check_consumed_files.py + python scripts/snapshots.py pr-body "$DATASET" --summary summary.json > pr.md + head -1 pr.md > pr-title.txt + tail -n +3 pr.md > pr-body.md + - name: Branch, commit, push, open or update the PR + env: + GH_TOKEN: ${{ secrets.QUANTECON_SERVICES_PAT || github.token }} + PAT_PRESENT: ${{ secrets.QUANTECON_SERVICES_PAT != '' }} + GH_REPO: ${{ github.repository }} + DATASET: ${{ matrix.snapshot.dataset }} + STEM: ${{ matrix.snapshot.stem }} + WHY: ${{ matrix.snapshot.why }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + branch="refresh/$STEM" + title=$(cat pr-title.txt) + { + cat pr-body.md + echo + echo "Why now: $WHY. Bytes changed: \`$bytes_changed\`. Run: $RUN_URL" + if [ "$PAT_PRESENT" != "true" ]; then + echo + echo "_Opened with the workflow token (no \`QUANTECON_SERVICES_PAT\` reached this repo), so the required \`consumed-files\` check will not start by itself — close and reopen this PR, or push to it, to trigger it._" + fi + } > body.md + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$branch" + git add -A lectures provenance CATALOG.md + if git diff --cached --quiet; then + echo "nothing to commit for $DATASET"; exit 0 + fi + git commit -q -m "$title" -m "Scheduled refresh. $WHY. Run: $RUN_URL" + git push --force-with-lease origin "$branch" + open=$(gh pr list --head "$branch" --state open --json number --jq '.[0].number // empty') + if [ -n "$open" ]; then + gh pr edit "$open" --title "$title" --body-file body.md + gh pr comment "$open" --body "Updated by a later run: $RUN_URL" + else + gh pr create --base main --head "$branch" --title "$title" --body-file body.md + fi diff --git a/AGENTS.md b/AGENTS.md index 9487e23..46f98db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,7 +118,9 @@ Where one builder produces a **set** of files, name it for the set and let each **Where a builder reads its input from.** The normal case is the third-party upstream, fetched at run time: eight of the nine `committed` builders here do that, and it is the fetch stage of the contract below. A builder reads from `sources/` **only when the input cannot be re-fetched** — the upstream is gone, unlocatable, or was inherited with no recoverable source. `sources/` is that exception layer, not a general input tree, and it is emphatically not "the big-file directory": the defining property is un-refetchability, not size. What it must never be is a network read from another QuantEcon repo — that is how a retired repo becomes load-bearing again. -Builders follow four stages — **fetch → pre-process → validate → write** — and only write on validation pass (expected columns/dtypes, row-count floor, recency of date range, no all-NaN columns, values unchanged in the overlap window with the previous vintage). Lectures always read the last-good snapshot: an upstream outage may fail a refresh, it must never break a lecture build. +Builders follow four stages — **fetch → pre-process → validate → write** — and only write on validation pass (expected columns/dtypes, row-count floor, recency of date range, no all-NaN columns, and a **bounded** overlap window against the previous vintage — a tracking snapshot is revised by its source, so the test is a tolerance plus a printed summary, never equality). Lectures always read the last-good snapshot: an upstream outage may fail a refresh, it must never break a lecture build. + +**A dynamic snapshot's builder also honours the refresh contract** that `.github/workflows/refresh-snapshots.yml` and `scripts/snapshots.py` rely on — copy `builders/_template.py`: `--out-dir` (dry run for the weekly canary), `--summary-json` (the run summary the manifest stamp and the refresh PR body are built from), writes through a temp file and `os.replace()`, and exit code **2** for a `ValidationError` against **1** for a fetch failure, which is how the canary issue tells "the data broke the contract" from "the network was down". The manifest fields the workflow stamps (`retrieved`, `integrity.sha256`, `integrity.upstream.*`, `schema.date_range.end`) must be single-line values with their reasoning in comments **above** them, not beside — the stamp replaces the line. ### Live APIs @@ -178,6 +180,8 @@ scripts/ # repo tooling — NOT published, produces no dataset build_catalog.py # generates CATALOG.md from the manifests build_audit.py # the audit dashboard: scan lecture repos → audit.json → site/ render_audit.py # its render stage + snapshots.py # dynamic snapshots: which are due, stamp a manifest after + # a refresh, render the refresh PR body audit_annotations.yml # curated judgment for not-yet-migrated data refs migration.yml # migration lifecycle tracker (status + PR provenance per dataset) manifest-schema.yml # per-dataset manifest schema (strawman) diff --git a/CATALOG.md b/CATALOG.md index 47f2c40..0ec2ca6 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -18,7 +18,7 @@ The dataset registry, **auto-generated** from the sidecar manifests (`lectures/* | [**assignat.xlsx**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/assignat.xlsx)
French Revolution — assignat issues, budgets and seigniorage (Sargent-Velde) | verbatim | [Sargent and Velde, "Macroeconomic Features of the French Revolution" — supporting spreadsheets](https://www.journals.uchicago.edu/doi/10.1086/261992) | | ✅ permitted | ⚠️ unverifiable | n/a (verbatim) | 204.6 KB | [lecture-python-intro · french_rev.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/french_rev.md)
[lecture-wasm · french_rev.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/french_rev.md)
[lecture-intro.zh-cn · french_rev.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/french_rev.md)
[test-actions-lecture-intro · french_rev.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/french_rev.md)
[tom-econ370-2025 · french_rev.md](https://github.com/QuantEcon/tom-econ370-2025/blob/main/lectures/french_rev.md)
⚠️ BROKEN reader (measured 2026-08-19): fetches this dataset through a stale `base_url` still pointing at lecture-python-intro's deleted `datasets/` copy (french_rev.md:70-75), which serves 404 | | [**bbh_macro_quarterly.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/bbh_macro_quarterly.csv)
Bhandari-Borovička-Ho replication — quarterly US macro series for the belief-wedge VAR, 1955Q1-2019Q4 | constructed | [Replication package for "Survey data and subjective beliefs in business cycle models" (Bhandari, Borovička and Ho), file `data input/FRED/data_FRED.xlsx`](https://doi.org/10.5281/zenodo.10194324) | CC-BY-4.0 | ✅ permitted | ✅ verified | ✅ committed | 31.5 KB | [lecture-python-advanced.myst · subjective_beliefs_business_cycles.md](https://github.com/QuantEcon/lecture-python-advanced.myst/blob/main/lectures/subjective_beliefs_business_cycles.md) | | [**bbh_michigan_monthly.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/bbh_michigan_monthly.csv)
Michigan Surveys of Consumers monthly aggregates and the US unemployment rate, 1978-01 to 2020-03 (BBH replication extract) | constructed | [Bhandari, Borovička and Ho replication package (Zenodo), carrying University of Michigan Surveys of Consumers published aggregates and a US Bureau of Labor Statistics series retrieved via FRED](https://doi.org/10.5281/zenodo.10194324) | CC-BY-4.0 | ⚠️ restricted | ✅ verified | ✅ committed | 11.9 KB | [lecture-python-advanced.myst · subjective_beliefs_business_cycles.md](https://github.com/QuantEcon/lecture-python-advanced.myst/blob/main/lectures/subjective_beliefs_business_cycles.md) | -| [**business_cycle_data.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/business_cycle_data.csv)
World Bank GDP growth (annual %) — USA, ARG, GBR, GRC, JPN, 1960 to 2023 | dynamic-snapshot | [World Bank, World Development Indicators (national accounts data, and OECD National Accounts data files)](https://data.worldbank.org/indicator/NY.GDP.MKTP.KD.ZG) | CC BY-4.0 | ✅ permitted | ⇄ diverged | ✅ committed | 5.8 KB | — | +| [**business_cycle_data.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/business_cycle_data.csv)
World Bank GDP growth (annual %) — USA, ARG, GBR, GRC, JPN, 1960 onward | dynamic-snapshot | [World Bank, World Development Indicators (national accounts data, and OECD National Accounts data files)](https://data.worldbank.org/indicator/NY.GDP.MKTP.KD.ZG) | CC BY-4.0 | ✅ permitted | ⇄ diverged | ✅ committed | 5.8 KB | — | | [**caron.npy**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/caron.npy)
French Revolution — monthly specie value of the assignat, 1791-1796 | constructed | unrecorded | | ✅ permitted | ⚠️ unverifiable | ⚠️ unrecovered | 1.1 KB | [lecture-python-intro · french_rev.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/french_rev.md)
[lecture-wasm · french_rev.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/french_rev.md)
[lecture-intro.zh-cn · french_rev.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/french_rev.md)
[test-actions-lecture-intro · french_rev.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/french_rev.md)
⚠️ Reads a local `datasets/` copy, not this file
[tom-econ370-2025 · french_rev.md](https://github.com/QuantEcon/tom-econ370-2025/blob/main/lectures/french_rev.md)
⚠️ Course fork with a live Pages site; reads its own blob-identical `datasets/` copy (french_rev.md:715-716), not this file, and its `base_url` still points at lecture-python-intro
[python-lecture-sandpit.myst · french_rev.md](https://github.com/QuantEcon/python-lecture-sandpit.myst/blob/main/lectures/french_rev.md)
⚠️ Public sandpit holding `lectures/_static/` copies | | [**chapter_3.xlsx**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/chapter_3.xlsx)
The Ends of Four Big Inflations — appendix tables, transcribed | constructed | [Sargent, "Rational Expectations and Inflation", chapter 3 appendix tables](https://press.princeton.edu/books/paperback/9780691158709/rational-expectations-and-inflation) | | ✅ permitted | ⚠️ unverifiable | ⚠️ unrecovered | 71.6 KB | [lecture-python-intro · inflation_history.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/inflation_history.md)
[lecture-wasm · inflation_history.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/inflation_history.md)
[lecture-intro.zh-cn · inflation_history.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/inflation_history.md)
[test-actions-lecture-intro · inflation_history.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/inflation_history.md) | | [**cities_brazil.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/cities_brazil.csv)
World Population Review — Brazilian city populations, 2023 | verbatim | [World Population Review — cities in Brazil](https://worldpopulationreview.com/countries/cities/brazil) | | ⚠️ restricted | ⚠️ unverifiable | n/a (verbatim) | 17.5 KB | [lecture-python-intro · heavy_tails.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/heavy_tails.md)
[lecture-wasm · heavy_tails.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/heavy_tails.md)
[lecture-intro.zh-cn · heavy_tails.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/heavy_tails.md)
[test-actions-lecture-intro · heavy_tails.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/heavy_tails.md) | diff --git a/PLAN.md b/PLAN.md index 795e8c1..20114c1 100644 --- a/PLAN.md +++ b/PLAN.md @@ -298,9 +298,9 @@ Full automation: - [x] Audit dashboard workflow ([#20](https://github.com/QuantEcon/data-lectures/issues/20), added 2026-07-17): `.github/workflows/audit-dashboard.yml` rebuilds the full-universe data audit + migration tracker from the 8 lecture repos' `main` (push to main / weekly / dispatch) and deploys it with the published tree to Pages. Strict mode fails the build on an unannotated data reference or a `migration.yml` status the scan contradicts - [ ] PR validation: manifest schema check + per-dataset invariant tests (expected columns/dtypes, row-count floor, date-range recency, no all-NaN columns, overlap-window agreement with the previous vintage) on every PR touching data. The schema decisions these tests force — column patterns for wide files, `known_nulls` exact-vs-ceiling, a canonical dtype vocabulary — are researched in [#14](https://github.com/QuantEcon/data-lectures/issues/14) - [x] Retrofit `builders/business_cycle.py` to the four-stage builder contract — **done 2026-09-01**, with the two provenance dumps moved out of the published tree to `provenance/` ([#13](https://github.com/QuantEcon/data-lectures/issues/13)). Its `validate()` is the first to face a *revised* upstream: it bounds the overlap window (5 pp) and prints the revision summary rather than asserting equality, which is the review surface the refresh-as-PR workflow below will use. It previously had fetch/transform/write but no validate stage. Builder architecture and a copy-able template: [#14](https://github.com/QuantEcon/data-lectures/issues/14) -- [ ] Scheduled refresh workflow for dynamic datasets — cron per cadence class, runs the builder (fetch → pre-process → validate → write), lands the result as a PR whose diff summary (rows added, date-range delta, overlap-window changes) is the review surface; low-risk series may auto-merge on green (first consumer: the UNRATE pilot, meta#338 P4) -- [ ] Weekly sources-alive canary: fetch + validate, no commit, opens an issue on failure — relocates API fragility from 7 lecture repos' CI into one scheduled job here -- [ ] Consumer fan-out: a merged refresh or in-place correction dispatches rebuilds of the repos in the dataset's machine-readable `consumers` list +- [x] Scheduled refresh workflow for dynamic datasets — **landed 2026-09-01** as `.github/workflows/refresh-snapshots.yml`, manifest-driven rather than cron-per-class: a weekly run asks `scripts/snapshots.py due` which `dynamic-snapshot` datasets have their cadence elapsed (or are `diverged`, or were never refreshed), runs each builder in place, stamps the manifest (`retrieved`, `sha256`, `integrity.upstream: verified`, `date_range.end`), regenerates the catalog, and opens a PR on `refresh/` whose body is the builder's overlap summary. Nothing auto-merges; the first consumer is `business_cycle_data.csv`, not UNRATE — the pilot's order inverted once the World Bank file turned out to be the one already here +- [x] Weekly sources-alive canary: fetch + validate, no commit, opens an issue on failure — **landed 2026-09-01** as the `canary` job of the same workflow: every dynamic snapshot's builder runs with `--out-dir`, and a failure opens or updates one `upstream-break` issue classified by exit code (2 = the data broke the contract, a human; anything else = the fetch, a retry). Covers the live APIs only as their snapshot twins land here — the 23 live-API lectures without a twin are still guarded by nothing but their own CI +- [ ] Consumer fan-out: a merged refresh or in-place correction dispatches rebuilds of the repos in the dataset's machine-readable `consumers` list — **policy recorded 2026-09-01** (AGENTS.md "Refresh, break, or schema change": per-consumer `on_refresh: rebuild | review`), and the refresh PR body already lists what the fan-out would do; the dispatch itself waits for the first snapshot with a consumer - [ ] Package the refresh job as a reusable workflow (`quantecon/actions`) once it stabilizes ### Phase 6 — Metadata backfill for existing holdings diff --git a/README.md b/README.md index d449334..4478297 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,11 @@ See the [draft convention](https://github.com/QuantEcon/QuantEcon.manual/pull/10 | Path | What | Published | | --- | --- | --- | -| `lectures/` | the published tree — flat. Every dataset lives here, directly. No folder implies ownership by a lecture series: any lecture may consume any file | yes | -| `scripts/` | builders for constructed and dynamic datasets, plus the audit-dashboard generator | no | +| `lectures/` | the published tree — flat. Every dataset lives here, directly, beside its sidecar manifest `.yml`. No folder implies ownership by a lecture series: any lecture may consume any file | yes | +| `builders/` | one builder per constructed or dynamic dataset, `builders/.` → `lectures/.` | no | +| `sources/` | builder inputs that cannot be re-fetched (per-path LFS); `sources/README.md` is their audit trail | no | +| `provenance/` | upstream metadata dumps a builder writes beside its dataset — evidence for the manifest's `source` and `license` fields, regenerated every run | no | +| `scripts/` | repo tooling: the catalog generator, the audit dashboard, and the dynamic-snapshot plumbing (`snapshots.py`) | no | | `manifest-schema.yml` | the per-dataset manifest schema (strawman — see [`PLAN.md`](PLAN.md) Phase 2) | no | | `migration.yml` | the migration lifecycle tracker — which PRs landed and repointed each dataset (transitional; archivable when the migration programme completes) | rendered | @@ -69,3 +72,21 @@ not-yet-migrated references). A new data reference with no annotation, or a migration status the scan contradicts, **fails the build** — the dashboard cannot silently rot. CI rebuilds it on push to `main`, weekly, and on demand (`.github/workflows/audit-dashboard.yml`). + +## Dynamic snapshots + +A `class: dynamic-snapshot` dataset tracks a moving source (World Bank, FRED) +and is refreshed **in place, by PR, never silently**. Every week +`.github/workflows/refresh-snapshots.yml` runs each such builder as a +**canary** (fetch + validate, no commit; a failure opens an `upstream-break` +issue here, and no lecture is affected because consumers read the last-good +snapshot), and for any dataset whose `cadence` has elapsed it runs the builder +for real, stamps the manifest, and opens a PR on `refresh/` whose body is +the builder's overlap summary — the place a revision is reviewed rather than +merely diffed. The rules for who is told what are in `AGENTS.md`, "Refresh, +break, or schema change". + +``` +python scripts/snapshots.py due # what would refresh this week +python builders/.py --out-dir /tmp/x --summary-json /tmp/s.json # dry run +``` diff --git a/builders/README.md b/builders/README.md index 874d0c6..ecbd38a 100644 --- a/builders/README.md +++ b/builders/README.md @@ -24,8 +24,19 @@ dataset claiming a builder names one. Builders follow four stages — **fetch → pre-process → validate → write** — and only write on validation pass. Lectures always read the last-good snapshot: an upstream outage may fail a refresh, it must never break a lecture build. The -template and the architecture discussion are in -[#14](https://github.com/QuantEcon/data-lectures/issues/14). +architecture discussion is in +[#14](https://github.com/QuantEcon/data-lectures/issues/14); the copy-able +template is [`_template.py`](_template.py) (not a builder — the underscore +keeps it out of any manifest). + +A **dynamic snapshot's** builder additionally honours the refresh contract +(`.github/workflows/refresh-snapshots.yml`, `scripts/snapshots.py`): +`--out-dir` for a dry run (the weekly canary), `--summary-json` for the run +summary the refresh PR and the manifest stamp are built from, atomic writes, +and exit code 2 for a `ValidationError` against 1 for a fetch failure. The +overlap window against the previous vintage is **bounded and reported, never +asserted equal** — the source revises the series; measure its routine +revisions before choosing the bound. Most builders fetch from the third-party upstream at run time, which is the normal case. A builder reads from `sources/` only when its input **cannot be diff --git a/builders/_template.py b/builders/_template.py new file mode 100644 index 0000000..c4af91c --- /dev/null +++ b/builders/_template.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +""" +TEMPLATE for a dynamic-snapshot builder -- copy to builders/.py and +replace every NotImplementedError. Not a builder itself: the leading +underscore keeps it out of any manifest's `builder:` field, and CI asserts +only the paths manifests name. + +Distilled from builders/business_cycle.py (the first dynamic snapshot) and +QuantEcon/data-lectures#14. The contract the refresh workflow relies on +(.github/workflows/refresh-snapshots.yml, scripts/snapshots.py): + + stages fetch -> pre_process -> validate -> write, writing ONLY on a + validation pass, through a temp file + os.replace(), so a + failed or interrupted refresh leaves the last-good snapshot + --out-dir dry run: write everything to a directory, still validating + against the committed file in lectures/ (the weekly canary) + --summary-json + write the run summary as JSON -- the refresh PR's body and the + manifest stamp are built from it; keys: dataset, builder, rows, + columns, date_range{start,end}, overlap{window, previous_end, + cells_total, cells_revised, max_abs_change, new_columns}|null + exit codes 0 ok; 1 fetch/other failure (retry); 2 ValidationError (the + data broke the contract -- a human decides) + overlap a TRACKING snapshot is revised by its source; validate() bounds + the overlap window and reports it rather than asserting + equality. Measure the source's routine revisions before + choosing the bound (business_cycle: observed max 1.5 pp, + bound 5 pp) + provenance upstream metadata dumps go to provenance/, never lectures/ + +Requires pandas plus whatever the source needs (add it to requirements.txt). +""" +import argparse +import datetime as dt +import json +import os +import sys + +import pandas as pd + +CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(CURRENT_FILE_DIR) +PUBLISHED_DIR = os.path.join(REPO_ROOT, 'lectures') +PROVENANCE_DIR = os.path.join(REPO_ROOT, 'provenance') + +OUT_FILE = '.csv' # lectures/.csv -- the manifest's filename +MAX_REVISION = None # overlap bound in the data's own units; measure first +MAX_STALENESS = None # newest observation must be at least this recent + + +class ValidationError(Exception): + """The fetched data broke the published contract -- exit code 2.""" + + +def _check(condition, message): + if not condition: + raise ValidationError(message) + + +def fetch(): + """Pull raw data from the upstream. Network failures surface here (exit 1).""" + raise NotImplementedError + + +def pre_process(raw): + """Pure raw -> published frame. No I/O; absorb upstream renames HERE so + the published schema never changes under a consumer.""" + raise NotImplementedError + + +def validate(frame, previous=None): + """Assert the contract the manifest's schema block promises; return the + run summary. Every failure is a ValidationError with a message a human + can act on from the canary issue.""" + _check(len(frame) > 0, 'empty frame') + # columns / dtypes / known_nulls / units / recency ... + summary = { + 'dataset': OUT_FILE, + 'builder': os.path.relpath(os.path.abspath(__file__), REPO_ROOT), + 'rows': int(frame.shape[0]), + 'columns': int(frame.shape[1]), + 'date_range': {'start': None, 'end': None}, + 'overlap': None, + } + if previous is not None: + # compare the shared window; report and BOUND revisions, never assert equality + raise NotImplementedError + return summary + + +def _atomic_write(path, text): + tmp = path + '.tmp' + with open(tmp, 'w') as f: + f.write(text) + os.replace(tmp, path) + + +def run(out_dir=None, summary_json=None): + data_dir = out_dir or PUBLISHED_DIR + previous_path = os.path.join(PUBLISHED_DIR, OUT_FILE) + previous = pd.read_csv(previous_path, index_col=0) if os.path.exists(previous_path) else None + + frame = pre_process(fetch()) + summary = validate(frame, previous) + if summary_json: + _atomic_write(summary_json, json.dumps(summary, indent=1) + '\n') + + os.makedirs(data_dir, exist_ok=True) + _atomic_write(os.path.join(data_dir, OUT_FILE), frame.to_csv()) + print(f'wrote {OUT_FILE}: {frame.shape[0]} rows x {frame.shape[1]} cols -> {data_dir}') + + +if __name__ == '__main__': + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('--out-dir') + ap.add_argument('--summary-json') + args = ap.parse_args() + try: + run(args.out_dir, args.summary_json) + except ValidationError as exc: + print(f'::error::{OUT_FILE}: validation failed -- {exc}', file=sys.stderr) + sys.exit(2) diff --git a/builders/business_cycle.py b/builders/business_cycle.py index 3cbad00..78075b2 100644 --- a/builders/business_cycle.py +++ b/builders/business_cycle.py @@ -37,17 +37,27 @@ snapshot in place. Usage: - python builders/business_cycle.py # refresh in place - python builders/business_cycle.py --out-dir D # write everything to D - # (a dry run; validates - # against lectures/ still) + python builders/business_cycle.py # refresh in place + python builders/business_cycle.py --out-dir D # dry run: write to D, + # still validating + # against lectures/ + python builders/business_cycle.py --summary-json S # also write the + # machine-readable + # run summary to S + +Exit codes (the contract .github/workflows/refresh-snapshots.yml relies on): + 0 wrote (or dry-ran) a validated snapshot + 1 fetch or other infrastructure failure -- retry, the data is not at fault + 2 ValidationError -- the fetched data broke the contract; needs a human Requires pandas and wbgapi (requirements.txt). """ import argparse import datetime as dt +import json import os import re +import sys import pandas as pd import wbgapi as wb @@ -82,6 +92,17 @@ MAX_STALENESS_YEARS = 2 +class ValidationError(Exception): + """The fetched data broke the published contract -- distinct from a fetch + failure, so CI can tell 'the data is wrong' (a human) from 'the network + was down' (a retry). Exit code 2.""" + + +def _check(condition, message): + if not condition: + raise ValidationError(message) + + def fetch(): """Live WDI, exactly the lecture's own three calls.""" frame = wb.data.DataFrame(SERIES, ECONOMIES, labels=True) @@ -105,49 +126,71 @@ def _years(frame): def validate(frame, previous=None): - """Refuse to write anything that is not the shape we expect.""" + """Refuse to write anything that is not the shape we expect. Returns the + run summary (the refresh PR's body and the manifest stamp are built from + it), raising ValidationError on the first broken invariant.""" # Grid: Country, then YR..YR with no gap. - assert list(frame.columns[:1]) == ['Country'], list(frame.columns[:3]) + _check(list(frame.columns[:1]) == ['Country'], f'first columns {list(frame.columns[:3])}') years = _years(frame) - assert len(years) == len(frame.columns) - 1, 'non-year column present' - assert years[0] == FIRST_YEAR, years[0] - assert years == list(range(FIRST_YEAR, years[-1] + 1)), 'gap in the year grid' + _check(len(years) == len(frame.columns) - 1, 'non-year column present') + _check(years[0] == FIRST_YEAR, f'first year is {years[0]}, not {FIRST_YEAR}') + _check(years == list(range(FIRST_YEAR, years[-1] + 1)), 'gap in the year grid') year_cols = [f'YR{y}' for y in years] # Economies: exactly the five, one row each. - assert frame.index.name == 'economy' - assert sorted(frame.index) == sorted(ECONOMIES), sorted(frame.index) - assert frame['Country'].notnull().all() + _check(frame.index.name == 'economy', f'index is {frame.index.name!r}') + _check(sorted(frame.index) == sorted(ECONOMIES), f'economies {sorted(frame.index)}') + _check(frame['Country'].notnull().all(), 'a Country label is missing') # Dtypes and units. values = frame[year_cols] - assert all(pd.api.types.is_float_dtype(values[c]) for c in year_cols), 'non-float year column' - assert values.abs().max().max() >= MIN_ABS_MAX, 'values look like ratios, not percent' - assert values.abs().max().max() <= MAX_ABS, 'growth rate out of band' + _check(all(pd.api.types.is_float_dtype(values[c]) for c in year_cols), 'non-float year column') + _check(values.abs().max().max() >= MIN_ABS_MAX, 'values look like ratios, not percent') + _check(values.abs().max().max() <= MAX_ABS, 'growth rate out of band') # The one structural null: growth is undefined in the series' first year. nulls = values.isnull().sum() - assert dict(nulls[nulls > 0]) == {f'YR{FIRST_YEAR}': len(ECONOMIES)}, dict(nulls[nulls > 0]) + _check(dict(nulls[nulls > 0]) == {f'YR{FIRST_YEAR}': len(ECONOMIES)}, + f'unexpected nulls {dict(nulls[nulls > 0])}') # Recency. - assert years[-1] >= dt.date.today().year - MAX_STALENESS_YEARS, f'newest year is {years[-1]}' + _check(years[-1] >= dt.date.today().year - MAX_STALENESS_YEARS, f'newest year is {years[-1]}') + + summary = { + 'dataset': OUT_FILE, + 'builder': os.path.relpath(os.path.abspath(__file__), REPO_ROOT), + 'rows': int(frame.shape[0]), + 'columns': int(frame.shape[1]), + 'date_range': {'start': years[0], 'end': years[-1]}, + 'overlap': None, + } # Overlap window against the last-good snapshot: revisions are expected, # bounded, and reported; a lost observation or a rescale is not. if previous is not None: prev_years = [f'YR{y}' for y in _years(previous)] - assert set(prev_years) <= set(year_cols), 'a year column disappeared' - assert sorted(previous.index) == sorted(frame.index), 'the economy set changed' + _check(set(prev_years) <= set(year_cols), 'a year column disappeared') + _check(sorted(previous.index) == sorted(frame.index), 'the economy set changed') old = previous.loc[frame.index, prev_years] new = frame.loc[frame.index, prev_years] - assert not (old.notnull() & new.isnull()).any().any(), 'a populated cell went empty' + _check(not (old.notnull() & new.isnull()).any().any(), 'a populated cell went empty') diff = (old - new).abs() changed = int((diff > 1e-9).sum().sum()) worst = float(diff.max().max()) + new_cols = sorted(set(year_cols) - set(prev_years)) + summary['overlap'] = { + 'window': f'{prev_years[0]}..{prev_years[-1]}', + 'previous_end': _years(previous)[-1], + 'cells_total': int(diff.size), + 'cells_revised': changed, + 'max_abs_change': round(worst, 4), + 'new_columns': new_cols, + } print(f'overlap window {prev_years[0]}..{prev_years[-1]}: ' f'{changed} of {diff.size} cells revised, max |change| {worst:.3f} pp; ' - f'new columns: {sorted(set(year_cols) - set(prev_years)) or "none"}') - assert worst <= MAX_REVISION, f'revision of {worst:.3f} pp exceeds {MAX_REVISION}' + f'new columns: {new_cols or "none"}') + _check(worst <= MAX_REVISION, f'revision of {worst:.3f} pp exceeds {MAX_REVISION}') + return summary def _atomic_write(path, text): @@ -159,7 +202,7 @@ def _atomic_write(path, text): os.replace(tmp, path) -def run(out_dir=None): +def run(out_dir=None, summary_json=None): data_dir = out_dir or PUBLISHED_DIR prov_dir = out_dir or PROVENANCE_DIR previous_path = os.path.join(PUBLISHED_DIR, OUT_FILE) @@ -168,7 +211,9 @@ def run(out_dir=None): frame, metadata, info = fetch() frame = pre_process(frame) - validate(frame, previous) + summary = validate(frame, previous) + if summary_json: + _atomic_write(summary_json, json.dumps(summary, indent=1) + '\n') os.makedirs(data_dir, exist_ok=True) os.makedirs(prov_dir, exist_ok=True) @@ -183,4 +228,10 @@ def run(out_dir=None): if __name__ == '__main__': ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) ap.add_argument('--out-dir', help='write outputs here instead of lectures/ and provenance/') - run(ap.parse_args().out_dir) + ap.add_argument('--summary-json', help='also write the run summary (JSON) here') + args = ap.parse_args() + try: + run(args.out_dir, args.summary_json) + except ValidationError as exc: + print(f'::error::{OUT_FILE}: validation failed -- {exc}', file=sys.stderr) + sys.exit(2) diff --git a/lectures/business_cycle_data.csv.yml b/lectures/business_cycle_data.csv.yml index a60e00e..d36847d 100644 --- a/lectures/business_cycle_data.csv.yml +++ b/lectures/business_cycle_data.csv.yml @@ -12,13 +12,14 @@ # vintage under a new name. filename: business_cycle_data.csv -title: World Bank GDP growth (annual %) — USA, ARG, GBR, GRC, JPN, 1960 to 2023 +title: World Bank GDP growth (annual %) — USA, ARG, GBR, GRC, JPN, 1960 onward description: > Annual real GDP growth (percent, constant local currency) for the United States, Argentina, the United Kingdom, Greece and Japan, extracted from World Bank WDI series NY.GDP.MKTP.KD.ZG in the wide layout wbgapi emits: one row per economy (ISO3 code as the index, country name as `Country`), one - `YR` column per year from 1960. It is the snapshot twin of the live + `YR` column per year from 1960 to the newest year the source carries + (`schema.date_range.end`). It is the snapshot twin of the live `wb.data.DataFrame` call the intro `business_cycle` lecture makes; the call is reproduced verbatim by the builder. @@ -62,18 +63,16 @@ license: is the model case for this repo's cache-with-attribution policy (AGENTS.md, "Licensing and attribution"). -retrieved: null # The bytes were produced by the committed - # builder, in QuantEcon/data commit b857c5c of - # 2025-02-16 — but that is when QuantEcon ran - # the script and committed the output, and - # AGENTS.md says not to reconstruct a - # retrieval date from git history. The vintage - # is pinned by content instead: YR2023 is the - # last column and is populated for all five - # economies, which places the export after the - # WDI release that first carried 2023 growth, - # and integrity.upstream records how far the - # live series has moved since. +# `retrieved` is stamped by scripts/snapshots.py on every merged refresh; null +# until the first one. The inherited bytes were produced by the committed +# builder in QuantEcon/data commit b857c5c of 2025-02-16 — but that is when +# QuantEcon ran the script and committed the output, and AGENTS.md says not to +# reconstruct a retrieval date from git history. The vintage is pinned by +# content instead: YR2023 is the last column and is populated for all five +# economies, which places the export after the WDI release that first carried +# 2023 growth, and integrity.upstream records how far the live series has +# moved since. +retrieved: null maintainer: QuantEcon # --------------------------------------------------------------------------- @@ -91,6 +90,9 @@ integrity: # aggregate is expected to. For a dynamic snapshot this is the normal # state between refreshes, not a defect — the refresh is the resolution, # and it is a deliberate PR, not something this manifest asks for. + # status / date / against / note are stamped by scripts/snapshots.py on + # every merged refresh (the delta block below is dropped at the same time, + # since a fresh snapshot is `verified` by construction). status: diverged date: 2026-09-01 against: builders/business_cycle.py # live WDI, lastupdated 2026-07-13 @@ -141,10 +143,10 @@ schema: row_count_floor: 5 # exact by construction: the builder asks for # five economies and validate() asserts the # set. Columns, not rows, are what grow. - date_range: {start: 1960, end: 2023} # `end` is the newest year column in - # the COMMITTED bytes; a refresh PR - # updates it, which is what makes the - # refresh diff self-describing. + # `end` is the newest year column in the COMMITTED bytes, stamped by + # scripts/snapshots.py on every refresh — which is what makes a refresh PR's + # manifest diff self-describing. + date_range: {start: 1960, end: 2023} # Growth is undefined in the series' first year, so YR1960 is empty for all # five economies and nothing else is. validate() asserts exactly this hole. diff --git a/scripts/snapshots.py b/scripts/snapshots.py new file mode 100644 index 0000000..46769ab --- /dev/null +++ b/scripts/snapshots.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Manifest-driven plumbing for dynamic snapshots (PLAN Phase 5). + +The builders do the fetching and validating; this script does everything the +refresh workflow needs to know or write that lives in the manifests: + + list every `class: dynamic-snapshot` dataset with a + runnable builder, as JSON (the canary matrix) + due the subset whose refresh is due, as JSON (the refresh + matrix): cadence elapsed since `retrieved`, or + `integrity.upstream.status: diverged`, or never + refreshed. `--all` makes every dataset due + stamp after a builder run, rewrite the sidecar manifest from + --summary S the builder's --summary-json: integrity.sha256 of the + new bytes, `retrieved`, integrity.upstream (verified, + today, the builder, one-line note; the `diverged` + delta block is dropped), schema.date_range.end + pr-body the refresh PR's title (first line) and body, from the + --summary S same summary plus the manifest's consumers + +`stamp` edits the manifest TEXT, not a parsed-and-re-dumped copy: the sidecars +carry their reasoning as comments, and PyYAML would throw every one of them +away. It finds a key by walking indentation, replaces the key line plus any +continuation lines with a single-line value, and then re-parses the file with +PyYAML to prove the result reads back as intended — a stamp that cannot be +verified fails rather than lands. + +Usage from the workflow (.github/workflows/refresh-snapshots.yml): + + python scripts/snapshots.py due + python builders/.py --summary-json summary.json + python scripts/snapshots.py stamp --summary summary.json + python scripts/snapshots.py pr-body --summary summary.json +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import pathlib +import re +import sys + +import yaml + +REPO = pathlib.Path(__file__).resolve().parents[1] +LECTURES = REPO / "lectures" + +CADENCE_DAYS = {"daily": 1, "weekly": 7, "monthly": 30, "quarterly": 91, "annual": 365} + + +# --------------------------------------------------------------------------- +# Reading +# --------------------------------------------------------------------------- + +def load_manifests() -> dict[str, dict]: + out = {} + for path in sorted(LECTURES.glob("*.yml")): + m = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + out[m.get("filename") or path.name[:-4]] = m + return out + + +def snapshots(manifests: dict[str, dict]) -> list[dict]: + """Every dynamic snapshot with a runnable builder.""" + rows = [] + for fname, m in manifests.items(): + if m.get("class") != "dynamic-snapshot": + continue + if m.get("builder_status") != "committed" or not m.get("builder"): + continue + rows.append({ + "dataset": fname, + "stem": pathlib.Path(fname).stem, + "builder": m["builder"], + "cadence": m.get("cadence"), + "retrieved": _iso(m.get("retrieved")), + "upstream_status": ((m.get("integrity") or {}).get("upstream") or {}).get("status"), + }) + return rows + + +def _iso(value) -> str | None: + if value is None: + return None + if isinstance(value, (dt.date, dt.datetime)): + return value.strftime("%Y-%m-%d") + return str(value) + + +def is_due(row: dict, today: dt.date) -> tuple[bool, str]: + if row["upstream_status"] == "diverged": + return True, "integrity.upstream.status is `diverged` — known to be behind the source" + if not row["retrieved"]: + return True, "never refreshed (`retrieved: null`)" + days = CADENCE_DAYS.get(row["cadence"]) + if days is None: + return True, f"cadence {row['cadence']!r} is not in {sorted(CADENCE_DAYS)} — treating as due" + last = dt.date.fromisoformat(row["retrieved"]) + age = (today - last).days + if age >= days: + return True, f"cadence `{row['cadence']}` elapsed: last retrieved {last}, {age} days ago" + return False, f"not due: last retrieved {last}, {age} of {days} days" + + +# --------------------------------------------------------------------------- +# Text-level manifest editing +# --------------------------------------------------------------------------- + +KEY_RE = re.compile(r"^(?P[ ]*)(?P[A-Za-z_][\w.-]*):(?P.*)$") + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _find_key(lines: list[str], path: list[str]) -> tuple[int, int]: + """(start, end) line span of the key at `path`, including continuation + lines; end is exclusive. Comment and blank lines inside a block are + skipped when descending, and a trailing run of them is NOT swallowed into + the span, so the reasoning above the next key survives an edit.""" + start, stop, depth_indent = 0, len(lines), -1 + for i, key in enumerate(path): + found = None + for n in range(start, stop): + line = lines[n] + if not line.strip() or line.lstrip().startswith("#"): + continue + ind = _indent(line) + if ind <= depth_indent: + break # left the parent block + m = KEY_RE.match(line) + if m and m.group("key") == key and (depth_indent < 0 and ind == 0 or ind > depth_indent): + if found is None: + found = n + if i < len(path) - 1: + depth_indent = ind + start = n + 1 + # the block ends at the next line at <= this indent + stop = next((k for k in range(n + 1, len(lines)) + if lines[k].strip() and not lines[k].lstrip().startswith("#") + and _indent(lines[k]) <= ind), len(lines)) + break + if found is None: + raise KeyError(".".join(path)) + key_line = found + key_indent = _indent(lines[key_line]) + end = key_line + 1 + while end < len(lines): + line = lines[end] + if not line.strip(): + break + if line.lstrip().startswith("#") and _indent(line) <= key_indent: + break + if _indent(line) <= key_indent and not line.lstrip().startswith("#"): + break + end += 1 + # do not swallow trailing comment lines that sit at the key's own indent + while end > key_line + 1 and lines[end - 1].lstrip().startswith("#"): + end -= 1 + return key_line, end + + +def set_scalar(lines: list[str], path: list[str], value: str) -> None: + a, b = _find_key(lines, path) + indent = " " * _indent(lines[a]) + lines[a:b] = [f"{indent}{path[-1]}: {value}"] + + +def drop_key(lines: list[str], path: list[str]) -> None: + try: + a, b = _find_key(lines, path) + except KeyError: + return + # take any comment lines immediately above that explain this key + while a > 0 and lines[a - 1].lstrip().startswith("#") and _indent(lines[a - 1]) == _indent(lines[a]): + a -= 1 + del lines[a:b] + + +def yaml_str(text: str) -> str: + """A double-quoted YAML scalar, safe for any one-line text.""" + return json.dumps(text, ensure_ascii=False) + + +def sha256(path: pathlib.Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + +def cmd_list(args) -> int: + print(json.dumps(snapshots(load_manifests()), indent=1)) + return 0 + + +def cmd_due(args) -> int: + today = dt.date.today() + rows = [] + for row in snapshots(load_manifests()): + due, why = (True, "forced with --all") if args.all else is_due(row, today) + if args.dataset and row["dataset"] != args.dataset: + continue + print(f"{row['dataset']}: {'DUE' if due else 'skip'} — {why}", file=sys.stderr) + if due: + rows.append({**row, "why": why}) + print(json.dumps(rows, indent=1)) + return 0 + + +def cmd_stamp(args) -> int: + summary = json.loads(pathlib.Path(args.summary).read_text()) + dataset = args.dataset + assert summary["dataset"] == dataset, (summary["dataset"], dataset) + data_path = LECTURES / dataset + manifest_path = LECTURES / f"{dataset}.yml" + today = dt.date.today().isoformat() + digest = sha256(data_path) + end = summary["date_range"]["end"] + + lines = manifest_path.read_text(encoding="utf-8").split("\n") + set_scalar(lines, ["retrieved"], today) + set_scalar(lines, ["integrity", "sha256"], digest) + set_scalar(lines, ["integrity", "upstream", "status"], "verified") + set_scalar(lines, ["integrity", "upstream", "date"], today) + set_scalar(lines, ["integrity", "upstream", "against"], summary["builder"]) + overlap = summary.get("overlap") or {} + note = (f"Refreshed {today} by the scheduled refresh: these bytes are the builder's " + f"validated output from the live source that day, so they are verified by " + f"construction. Overlap window {overlap.get('window', 'n/a')}: " + f"{overlap.get('cells_revised', 0)} of {overlap.get('cells_total', 0)} cells " + f"revised, max |change| {overlap.get('max_abs_change', 0)}.") + set_scalar(lines, ["integrity", "upstream", "note"], yaml_str(note)) + for key in ("delta_kind", "delta", "delta_evidence", "register"): + drop_key(lines, ["integrity", "upstream", key]) + a, b = _find_key(lines, ["schema", "date_range"]) + lines[a] = re.sub(r"end: [^,}]+", f"end: {end}", lines[a]) + text = "\n".join(lines) + + # Prove it reads back as intended before it lands. + m = yaml.safe_load(text) + up = m["integrity"]["upstream"] + checks = { + "retrieved": _iso(m["retrieved"]) == today, + "sha256": m["integrity"]["sha256"] == digest, + "status": up["status"] == "verified", + "date": _iso(up["date"]) == today, + "against": up["against"] == summary["builder"], + "delta dropped": not any(k in up for k in ("delta_kind", "delta", "delta_evidence", "register")), + "date_range.end": m["schema"]["date_range"]["end"] == end, + } + failed = [k for k, ok in checks.items() if not ok] + if failed: + print(f"::error::{dataset}.yml: stamp did not read back — {failed}", file=sys.stderr) + return 1 + manifest_path.write_text(text, encoding="utf-8") + print(f"stamped {manifest_path.name}: retrieved {today}, sha256 {digest[:12]}…, end {end}") + return 0 + + +def cmd_pr_body(args) -> int: + summary = json.loads(pathlib.Path(args.summary).read_text()) + dataset = args.dataset + m = load_manifests()[dataset] + ov = summary.get("overlap") or {} + end = summary["date_range"]["end"] + prev_end = ov.get("previous_end") + title = (f"Refresh {dataset}: {prev_end} → {end}" if prev_end and prev_end != end + else f"Refresh {dataset} ({end} vintage, values revised)") + consumers = m.get("consumers") or [] + lines = [ + title, + "", + f"Scheduled refresh of `{dataset}` (`class: dynamic-snapshot`, `cadence: {m.get('cadence')}`) " + f"by `{summary['builder']}` on {dt.date.today().isoformat()}. The builder fetched the live source, " + f"validated the result against the published contract, and wrote it; the manifest is stamped to match " + f"(`retrieved`, `integrity.sha256`, `integrity.upstream: verified`, `schema.date_range.end`) and " + f"`CATALOG.md` is regenerated. Review this PR through the overlap summary below — it is the one place a " + f"revision is a decision rather than a diff.", + "", + "| | |", + "| --- | --- |", + f"| Shape | {summary['rows']} rows × {summary['columns']} columns |", + f"| Date range | {summary['date_range']['start']} to {end}" + (f" (was {prev_end})" if prev_end else "") + " |", + ] + if ov: + lines += [ + f"| Overlap window | {ov['window']}: **{ov['cells_revised']} of {ov['cells_total']} cells revised**, max change {ov['max_abs_change']} |", + f"| New columns | {', '.join(ov['new_columns']) if ov['new_columns'] else 'none'} |", + ] + lines += ["", "**Consumers** (from the manifest; AGENTS.md \"Refresh, break, or schema change\"):", ""] + if consumers: + for c in consumers: + action = c.get("on_refresh", "unset") + lines.append(f"- `{c.get('repo')}` — `{c.get('file')}` — `on_refresh: {action}`" + + (" → open an issue there with this summary" if action == "review" + else " → dispatch a rebuild" if action == "rebuild" + else " → decide, then record `on_refresh` in the manifest")) + else: + lines.append("- none recorded — nothing to rebuild or notify.") + lines += ["", "_Opened automatically by `.github/workflows/refresh-snapshots.yml`. A later run of the same " + "refresh updates this branch and PR rather than opening another._"] + print("\n".join(lines)) + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("list").set_defaults(fn=cmd_list) + p = sub.add_parser("due"); p.add_argument("--all", action="store_true"); p.add_argument("--dataset") + p.set_defaults(fn=cmd_due) + p = sub.add_parser("stamp"); p.add_argument("dataset"); p.add_argument("--summary", required=True) + p.set_defaults(fn=cmd_stamp) + p = sub.add_parser("pr-body"); p.add_argument("dataset"); p.add_argument("--summary", required=True) + p.set_defaults(fn=cmd_pr_body) + args = ap.parse_args() + return args.fn(args) + + +if __name__ == "__main__": + sys.exit(main()) From 93e4a5c6b89c83ef0407c4914fb44c20bbca4572 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Tue, 1 Sep 2026 12:39:42 +1000 Subject: [PATCH 2/2] Copilot review on #110: no sourcing of generated files, no assert as a guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The canary now records its failure as JSON via jq and the notifier reads each field with jq -r — data is never evaluated as shell, in a job that holds issues: write. And snapshots.py's dataset-mismatch guard is an explicit check that emits ::error and exits 1, since python -O would have dropped the assert that stood there. Co-Authored-By: Claude Fable 5 --- .github/workflows/refresh-snapshots.yml | 22 ++++++++++++---------- scripts/snapshots.py | 7 ++++++- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/refresh-snapshots.yml b/.github/workflows/refresh-snapshots.yml index 63a0361..683afe6 100644 --- a/.github/workflows/refresh-snapshots.yml +++ b/.github/workflows/refresh-snapshots.yml @@ -90,7 +90,7 @@ jobs: snapshot: ${{ fromJson(needs.plan.outputs.all) }} name: canary (${{ matrix.snapshot.dataset }}) # A matrix cannot expose one output per leg, so a failing leg leaves its - # failure.env + log as an artifact and the notify job reads those. + # failure.json + log as an artifact and the notify job reads those. steps: - uses: actions/checkout@v4 with: @@ -115,12 +115,11 @@ jobs: *) kind=fetch ;; esac echo "kind=$kind" >> "$GITHUB_OUTPUT" - { - echo "dataset=$DATASET" - echo "builder=$BUILDER" - echo "kind=$kind" - echo "code=$code" - } > failure.env + # Data, not shell: the notifier reads this with jq, never sources it. + jq -n --arg dataset "$DATASET" --arg builder "$BUILDER" \ + --arg kind "$kind" --arg code "$code" \ + '{dataset: $dataset, builder: $builder, kind: $kind, code: $code}' \ + > failure.json exit "$code" - name: Keep the failure for the notifier if: failure() @@ -128,7 +127,7 @@ jobs: with: name: canary-failure-${{ matrix.snapshot.stem }} path: | - failure.env + failure.json builder.log retention-days: 7 @@ -156,8 +155,11 @@ jobs: echo "The weekly sources-alive canary failed. Nothing was written: every consumer keeps reading the last-good snapshot, so no lecture is affected — this is the alarm moving from the lecture repos' CI to here, doing its job." echo for dir in failures/*/; do - [ -f "$dir/failure.env" ] || continue - . "$dir/failure.env" + [ -f "$dir/failure.json" ] || continue + dataset=$(jq -r .dataset "$dir/failure.json") + builder=$(jq -r .builder "$dir/failure.json") + kind=$(jq -r .kind "$dir/failure.json") + code=$(jq -r .code "$dir/failure.json") case "$kind" in validation) what="**validation failed** (exit 2) — the fetched data broke the published contract. A human decides: absorb the upstream change in the builder's \`pre_process\` stage so the published schema is unchanged, or, if it cannot honestly be absorbed, plan a new-filename vintage (AGENTS.md, \"Refresh, break, or schema change\")." ;; *) what="**fetch failed** (exit $code) — the upstream or the network, not the data. Re-run the workflow; if it fails again, the source has moved or gone." ;; diff --git a/scripts/snapshots.py b/scripts/snapshots.py index 46769ab..9e98cd1 100644 --- a/scripts/snapshots.py +++ b/scripts/snapshots.py @@ -218,7 +218,12 @@ def cmd_due(args) -> int: def cmd_stamp(args) -> int: summary = json.loads(pathlib.Path(args.summary).read_text()) dataset = args.dataset - assert summary["dataset"] == dataset, (summary["dataset"], dataset) + if summary.get("dataset") != dataset: + # Not an assert: `python -O` would drop it, and this is the guard that + # stops one builder's summary from stamping another dataset's manifest. + print(f"::error::{dataset}: summary is for {summary.get('dataset')!r}, " + f"not {dataset!r} — refusing to stamp", file=sys.stderr) + return 1 data_path = LECTURES / dataset manifest_path = LECTURES / f"{dataset}.yml" today = dt.date.today().isoformat()