From e849b3a388f6f396583bf6045f1e2f560c3cf04f Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Tue, 1 Sep 2026 15:45:07 +1000 Subject: [PATCH 1/2] =?UTF-8?q?P4:=20the=20business=5Fcycle=20snapshot=20s?= =?UTF-8?q?et=20=E2=80=94=20three=20World=20Bank=20tables=20and=20a=20FRED?= =?UTF-8?q?=20composite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 of re-enabling business_cycle in lecture-wasm (lecture-wasm#70): every live read the lecture makes now has a snapshot here, with intro keeping its live calls as the lesson. - builders/_fred.py: the shared Fred class (fredgraph.csv over urllib, DATE index, `.` -> NaN, server-side frequency change). No pandas_datareader in a builder — the decision on #26. - builders/business_cycle_fred.py -> lectures/us_business_cycle_monthly.csv: UNRATE, USREC, UMCSENT, CPILFESL, INDPRO and the 1929-42 NBER unemployment series on one monthly grid from 1919-01. validate() places every null exactly (series starts, UMCSENT's sparse pre-1978 years, the 2025-10 shutdown hole in UNRATE and CPILFESL) and bounds revisions per series. cadence: monthly — the first. - builders/business_cycle.py now writes three tables from one WDI fetch: business_cycle_data.csv grown from five economies to the nine the lecture plots (free while it has no consumer), unemployment_rate_annual.csv (USA FRA GBR JPN) and private_credit_to_gdp.csv (GBR). Nulls are placed, not counted: only before an economy's first observation or in the newest two years. All three validate before any writes. --summary-json emits a list. - scripts/snapshots.py accepts a list summary and compares a date-shaped date_range.end as text (the stamp's read-back guard caught the mismatch). - Manifests for the three new files (names provisional pending #113), business_cycle_data.csv.yml updated for nine economies, all four stamped by snapshots.py (retrieved, sha256, verified, date_range.end); migration.yml records (landed, P4); CATALOG.md regenerated; PLAN P4 box reframed; builders/README and AGENTS record the per-source rule. UMCSENT's licence is recorded from FRED's own classification ("copyrighted: citation required") as permitted-with-attribution, the condition this repo's policy meets. Gates: check_consumed_files 44/0; strict audit adds no problems; all four snapshots list as verified and not due. See #26, #113, #14; QuantEcon/lecture-wasm#70. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 + CATALOG.md | 7 +- PLAN.md | 2 +- builders/README.md | 10 +- builders/_fred.py | 71 ++ builders/business_cycle.py | 265 ++-- builders/business_cycle_fred.py | 192 +++ lectures/business_cycle_data.csv | 4 + lectures/business_cycle_data.csv.yml | 64 +- lectures/private_credit_to_gdp.csv | 2 + lectures/private_credit_to_gdp.csv.yml | 79 ++ lectures/unemployment_rate_annual.csv | 5 + lectures/unemployment_rate_annual.csv.yml | 90 ++ lectures/us_business_cycle_monthly.csv | 1292 ++++++++++++++++++++ lectures/us_business_cycle_monthly.csv.yml | 131 ++ migration.yml | 33 + scripts/snapshots.py | 17 +- 17 files changed, 2085 insertions(+), 181 deletions(-) create mode 100644 builders/_fred.py create mode 100644 builders/business_cycle_fred.py create mode 100644 lectures/private_credit_to_gdp.csv create mode 100644 lectures/private_credit_to_gdp.csv.yml create mode 100644 lectures/unemployment_rate_annual.csv create mode 100644 lectures/unemployment_rate_annual.csv.yml create mode 100644 lectures/us_business_cycle_monthly.csv create mode 100644 lectures/us_business_cycle_monthly.csv.yml diff --git a/AGENTS.md b/AGENTS.md index ea959de..a05586a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,6 +120,8 @@ Where one builder produces a **set** of files, name it for the set and let each Builders follow four stages — **fetch → pre-process → validate → write** — and only write on validation pass (expected columns/dtypes, row-count floor, recency of date range, no all-NaN columns, and a **bounded** overlap window against the previous vintage — a tracking snapshot is revised by its source, so the test is a tolerance plus a printed summary, never equality). Lectures always read the last-good snapshot: an upstream outage may fail a refresh, it must never break a lecture build. +**One builder per source, composite files where a lecture reads series together, shared fetch libraries** (decided 2026-09-01, #26): `builders/_fred.py` is the `Fred` class for FRED reads — no `pandas_datareader` in a builder — and a lecture's FRED data is one file with the lecture's own variable names as columns, not six. A builder that writes a set validates every file before writing any. + **A dynamic snapshot's builder also honours the refresh contract** that `.github/workflows/refresh-snapshots.yml` and `scripts/snapshots.py` rely on — copy `builders/_template.py`: `--out-dir` (dry run for the weekly canary), `--summary-json` (the run summary the manifest stamp and the refresh PR body are built from), writes through a temp file and `os.replace()`, and exit code **2** for a `ValidationError` against **1** for a fetch failure, which is how the canary issue tells "the data broke the contract" from "the network was down". The manifest fields the workflow stamps (`retrieved`, `integrity.sha256`, `integrity.upstream.*`, `schema.date_range.end`) must be single-line values with their reasoning in comments **above** them, not beside — the stamp replaces the line. And **no prose in the manifest may embed a fact a refresh can change** — an end year, an observed range, a row count — because nothing re-writes prose: the first refresh PR (#112) shipped a column description still saying "YR2023 in the committed bytes". ### Live APIs diff --git a/CATALOG.md b/CATALOG.md index e8f6485..7e1653b 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -6,7 +6,7 @@ The dataset registry, **auto-generated** from the sidecar manifests (`lectures/*.yml`). Do not edit by hand — run `python scripts/build_catalog.py`. A dataset appears here once it has a manifest, which may be before its consuming lectures are repointed — an empty **Used by** column means the file is here and documented but no lecture reads it from this repo yet. Files still to migrate are tracked in [PLAN.md](PLAN.md). -**41 datasets** · 40 read by lectures today, 1 awaiting repoint · 113.2 MB total · 35 permitted / 6 restricted redistribution +**44 datasets** · 40 read by lectures today, 4 awaiting repoint · 113.2 MB total · 38 permitted / 6 restricted redistribution | Dataset | Class | Source | Licence | Redist. | Integrity | Builder | Size | Used by | | --- | --- | --- | --- | --- | --- | --- | --- | --- | @@ -18,7 +18,7 @@ The dataset registry, **auto-generated** from the sidecar manifests (`lectures/* | [**assignat.xlsx**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/assignat.xlsx)
French Revolution — assignat issues, budgets and seigniorage (Sargent-Velde) | verbatim | [Sargent and Velde, "Macroeconomic Features of the French Revolution" — supporting spreadsheets](https://www.journals.uchicago.edu/doi/10.1086/261992) | | ✅ permitted | ⚠️ unverifiable | n/a (verbatim) | 204.6 KB | [lecture-python-intro · french_rev.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/french_rev.md)
[lecture-wasm · french_rev.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/french_rev.md)
[lecture-intro.zh-cn · french_rev.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/french_rev.md)
[test-actions-lecture-intro · french_rev.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/french_rev.md)
[tom-econ370-2025 · french_rev.md](https://github.com/QuantEcon/tom-econ370-2025/blob/main/lectures/french_rev.md)
⚠️ BROKEN reader (measured 2026-08-19): fetches this dataset through a stale `base_url` still pointing at lecture-python-intro's deleted `datasets/` copy (french_rev.md:70-75), which serves 404 | | [**bbh_macro_quarterly.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/bbh_macro_quarterly.csv)
Bhandari-Borovička-Ho replication — quarterly US macro series for the belief-wedge VAR, 1955Q1-2019Q4 | constructed | [Replication package for "Survey data and subjective beliefs in business cycle models" (Bhandari, Borovička and Ho), file `data input/FRED/data_FRED.xlsx`](https://doi.org/10.5281/zenodo.10194324) | CC-BY-4.0 | ✅ permitted | ✅ verified | ✅ committed | 31.5 KB | [lecture-python-advanced.myst · subjective_beliefs_business_cycles.md](https://github.com/QuantEcon/lecture-python-advanced.myst/blob/main/lectures/subjective_beliefs_business_cycles.md) | | [**bbh_michigan_monthly.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/bbh_michigan_monthly.csv)
Michigan Surveys of Consumers monthly aggregates and the US unemployment rate, 1978-01 to 2020-03 (BBH replication extract) | constructed | [Bhandari, Borovička and Ho replication package (Zenodo), carrying University of Michigan Surveys of Consumers published aggregates and a US Bureau of Labor Statistics series retrieved via FRED](https://doi.org/10.5281/zenodo.10194324) | CC-BY-4.0 | ⚠️ restricted | ✅ verified | ✅ committed | 11.9 KB | [lecture-python-advanced.myst · subjective_beliefs_business_cycles.md](https://github.com/QuantEcon/lecture-python-advanced.myst/blob/main/lectures/subjective_beliefs_business_cycles.md) | -| [**business_cycle_data.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/business_cycle_data.csv)
World Bank GDP growth (annual %) — USA, ARG, GBR, GRC, JPN, 1960 onward | dynamic-snapshot | [World Bank, World Development Indicators (national accounts data, and OECD National Accounts data files)](https://data.worldbank.org/indicator/NY.GDP.MKTP.KD.ZG) | CC BY-4.0 | ✅ permitted | ✅ verified | ✅ committed | 6.0 KB | — | +| [**business_cycle_data.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/business_cycle_data.csv)
World Bank GDP growth (annual %) — nine economies, 1960 onward | dynamic-snapshot | [World Bank, World Development Indicators (national accounts data, and OECD National Accounts data files)](https://data.worldbank.org/indicator/NY.GDP.MKTP.KD.ZG) | CC BY-4.0 | ✅ permitted | ✅ verified | ✅ committed | 10.4 KB | — | | [**caron.npy**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/caron.npy)
French Revolution — monthly specie value of the assignat, 1791-1796 | constructed | unrecorded | | ✅ permitted | ⚠️ unverifiable | ⚠️ unrecovered | 1.1 KB | [lecture-python-intro · french_rev.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/french_rev.md)
[lecture-wasm · french_rev.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/french_rev.md)
[lecture-intro.zh-cn · french_rev.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/french_rev.md)
[test-actions-lecture-intro · french_rev.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/french_rev.md)
⚠️ Reads a local `datasets/` copy, not this file
[tom-econ370-2025 · french_rev.md](https://github.com/QuantEcon/tom-econ370-2025/blob/main/lectures/french_rev.md)
⚠️ Course fork with a live Pages site; reads its own blob-identical `datasets/` copy (french_rev.md:715-716), not this file, and its `base_url` still points at lecture-python-intro
[python-lecture-sandpit.myst · french_rev.md](https://github.com/QuantEcon/python-lecture-sandpit.myst/blob/main/lectures/french_rev.md)
⚠️ Public sandpit holding `lectures/_static/` copies | | [**chapter_3.xlsx**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/chapter_3.xlsx)
The Ends of Four Big Inflations — appendix tables, transcribed | constructed | [Sargent, "Rational Expectations and Inflation", chapter 3 appendix tables](https://press.princeton.edu/books/paperback/9780691158709/rational-expectations-and-inflation) | | ✅ permitted | ⚠️ unverifiable | ⚠️ unrecovered | 71.6 KB | [lecture-python-intro · inflation_history.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/inflation_history.md)
[lecture-wasm · inflation_history.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/inflation_history.md)
[lecture-intro.zh-cn · inflation_history.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/inflation_history.md)
[test-actions-lecture-intro · inflation_history.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/inflation_history.md) | | [**cities_brazil.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/cities_brazil.csv)
World Population Review — Brazilian city populations, 2023 | verbatim | [World Population Review — cities in Brazil](https://worldpopulationreview.com/countries/cities/brazil) | | ⚠️ restricted | ⚠️ unverifiable | n/a (verbatim) | 17.5 KB | [lecture-python-intro · heavy_tails.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/heavy_tails.md)
[lecture-wasm · heavy_tails.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/heavy_tails.md)
[lecture-intro.zh-cn · heavy_tails.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/heavy_tails.md)
[test-actions-lecture-intro · heavy_tails.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/heavy_tails.md) | @@ -47,9 +47,12 @@ The dataset registry, **auto-generated** from the sidecar manifests (`lectures/* | [**maketable4.dta**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/maketable4.dta)
Acemoglu-Johnson-Robinson (2001) colonial origins — table 4 replication data | verbatim | [Daron Acemoglu data archive — "The Colonial Origins of Comparative Development", table 4 (program and data files)](https://economics.mit.edu/people/faculty/daron-acemoglu/data-archive) | | ✅ permitted | ✅ verified | n/a (verbatim) | 11.7 KB | [lecture-python.myst · ols.md](https://github.com/QuantEcon/lecture-python.myst/blob/main/lectures/ols.md)
[lecture-python.zh-cn · ols.md](https://github.com/QuantEcon/lecture-python.zh-cn/blob/main/lectures/ols.md) | | [**mpd2020.xlsx**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/mpd2020.xlsx)
Maddison Project Database 2020 — GDP per capita and population, 1 CE to 2018 | constructed | [Maddison Project Database, version 2020](https://www.rug.nl/ggdc/historicaldevelopment/maddison/releases/maddison-project-database-2020) | CC BY 4.0 | ✅ permitted | ⇄ diverged | ⚠️ unrecovered | 1.7 MB | [lecture-python-intro · long_run_growth.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/long_run_growth.md)
[lecture-wasm · long_run_growth.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/long_run_growth.md)
[lecture-intro.zh-cn · long_run_growth.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/long_run_growth.md)
[test-actions-lecture-intro · long_run_growth.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/long_run_growth.md) | | [**nom_balances.npy**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/nom_balances.npy)
French Revolution — monthly nominal assignat balances, 1789-1796 | constructed | unrecorded | | ✅ permitted | ⚠️ unverifiable | ⚠️ unrecovered | 1.4 KB | [lecture-python-intro · french_rev.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/french_rev.md)
[lecture-wasm · french_rev.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/french_rev.md)
[lecture-intro.zh-cn · french_rev.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/french_rev.md)
[test-actions-lecture-intro · french_rev.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/french_rev.md)
⚠️ Reads a local `datasets/` copy, not this file
[tom-econ370-2025 · french_rev.md](https://github.com/QuantEcon/tom-econ370-2025/blob/main/lectures/french_rev.md)
⚠️ Course fork with a live Pages site; reads its own blob-identical `datasets/` copy (french_rev.md:715-716), not this file, and its `base_url` still points at lecture-python-intro
[python-lecture-sandpit.myst · french_rev.md](https://github.com/QuantEcon/python-lecture-sandpit.myst/blob/main/lectures/french_rev.md)
⚠️ Public sandpit holding `lectures/_static/` copies | +| [**private_credit_to_gdp.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/private_credit_to_gdp.csv)
World Bank domestic credit to the private sector (% of GDP) — United Kingdom, 1960 onward | dynamic-snapshot | [World Bank, World Development Indicators (International Monetary Fund, International Financial Statistics, and World Bank and OECD GDP estimates)](https://data.worldbank.org/indicator/FS.AST.PRVT.GD.ZS) | CC BY-4.0 | ✅ permitted | ✅ verified | ✅ committed | 1.6 KB | — | | [**realwage.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/realwage.csv)
OECD real minimum wages — 32 countries, 2006–2016 | constructed | [OECD — Real minimum wages (RMW)](https://stats.oecd.org/Index.aspx?DataSetCode=RMW) | CC BY 4.0 | ✅ permitted | ⚠️ unverifiable | ⚠️ unrecovered | 118.7 KB | [lecture-python-programming · pandas_panel.md](https://github.com/QuantEcon/lecture-python-programming/blob/main/lectures/pandas_panel.md)
[lecture-python.myst · pandas_panel.md](https://github.com/QuantEcon/lecture-python.myst/blob/main/lectures/pandas_panel.md)
[lecture-python.zh-cn · pandas_panel.md](https://github.com/QuantEcon/lecture-python.zh-cn/blob/main/lectures/pandas_panel.md) | | [**test_pwt.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/test_pwt.csv)
Penn World Table teaching extract — eight countries, year 2000 | constructed | [Penn World Table (Heston, Summers and Aten; PWT 6.x/7.x era) — exact vintage unestablished](https://www.rug.nl/ggdc/productivity/pwt/pwt-releases/pwt-7.0) | | ✅ permitted | ⚠️ unverifiable | ⚠️ unrecovered | 793 B | [lecture-python-programming · pandas.md](https://github.com/QuantEcon/lecture-python-programming/blob/main/lectures/pandas.md)
[lecture-python-programming · polars.md](https://github.com/QuantEcon/lecture-python-programming/blob/main/lectures/polars.md)
[lecture-python-programming.zh-cn · pandas.md](https://github.com/QuantEcon/lecture-python-programming.zh-cn/blob/main/lectures/pandas.md)
[lecture-python-programming.zh-cn · polars.md](https://github.com/QuantEcon/lecture-python-programming.zh-cn/blob/main/lectures/polars.md)
[lecture-python-programming.fr · pandas.md](https://github.com/QuantEcon/lecture-python-programming.fr/blob/main/lectures/pandas.md)
[lecture-python-programming.fr · polars.md](https://github.com/QuantEcon/lecture-python-programming.fr/blob/main/lectures/polars.md)
[lecture-python-programming.fa · pandas.md](https://github.com/QuantEcon/lecture-python-programming.fa/blob/main/lectures/pandas.md)
[lecture-python-programming.fa · polars.md](https://github.com/QuantEcon/lecture-python-programming.fa/blob/main/lectures/polars.md) | +| [**unemployment_rate_annual.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/unemployment_rate_annual.csv)
World Bank unemployment rate (annual, % of labour force) — USA, FRA, GBR, JPN, 1960 onward | dynamic-snapshot | [World Bank, World Development Indicators (International Labour Organization, ILOSTAT database, national estimates)](https://data.worldbank.org/indicator/SL.UEM.TOTL.NE.ZS) | CC BY-4.0 | ✅ permitted | ✅ verified | ✅ committed | 1.8 KB | — | | [**us_adult_heights.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/us_adult_heights.csv)
United States — adult standing height by sex, NHANES 2015-2018 | constructed | [National Health and Nutrition Examination Survey (NHANES), US Centers for Disease Control and Prevention, National Center for Health Statistics](https://www.cdc.gov/nchs/nhanes/index.htm) | US Government work — public domain | ✅ permitted | ✅ verified | ✅ committed | 123.1 KB | [lecture-python-intro · prob_dist.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/prob_dist.md)
[lecture-python-intro · observed_distributions.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/observed_distributions.md)
[lecture-python-intro · fitting_distributions.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/fitting_distributions.md)
[lecture-intro.zh-cn · fitting_distributions.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/fitting_distributions.md)
[lecture-intro.zh-cn · observed_distributions.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/observed_distributions.md)
[lecture-intro.zh-cn · prob_dist.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/prob_dist.md)
⚠️ Reads a local `_static/lecture_specific/prob_dist/` copy, not this file, under a comment that still says to switch once the datasets repo exists | +| [**us_business_cycle_monthly.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/us_business_cycle_monthly.csv)
US business-cycle indicators, monthly — unemployment, NBER recessions, consumer sentiment, core CPI, industrial production, 1919 onward | dynamic-snapshot | [FRED (Federal Reserve Bank of St. Louis) — series UNRATE, USREC, UMCSENT, CPILFESL, INDPRO, M0892AUSM156SNBR](https://fred.stlouisfed.org) | | ✅ permitted | ✅ verified | ✅ committed | 41.7 KB | — | | [**usa-gini-nwealth-tincome-lincome.csv**](https://github.com/QuantEcon/data-lectures/raw/main/lectures/usa-gini-nwealth-tincome-lincome.csv)
US Gini coefficients — net wealth, total income and labour income, 1950-2016 | constructed | [Derived from SCF_plus_mini.csv (this repo), an extract of the SCF+ panel](https://github.com/QuantEcon/data-lectures/blob/main/lectures/SCF_plus_mini.csv) | | ✅ permitted | ⚠️ unverifiable | committed-frozen | 1.2 KB | [lecture-python-intro · inequality.md](https://github.com/QuantEcon/lecture-python-intro/blob/main/lectures/inequality.md)
[lecture-wasm · inequality.md](https://github.com/QuantEcon/lecture-wasm/blob/main/lectures/inequality.md)
[lecture-intro.zh-cn · inequality.md](https://github.com/QuantEcon/lecture-intro.zh-cn/blob/main/lectures/inequality.md)
[test-actions-lecture-intro · inequality.md](https://github.com/QuantEcon/test-actions-lecture-intro/blob/main/lectures/inequality.md) | --- diff --git a/PLAN.md b/PLAN.md index 20114c1..c5a9d57 100644 --- a/PLAN.md +++ b/PLAN.md @@ -340,7 +340,7 @@ The first end-to-end deployment: one dataset per hosting pattern, each the harde **The flip was the acceptance test, and it measured as one.** Dry-run locally in both directions before pushing: `landed` → exit 1 with 6 warnings; `repointed` → exit 0. So the red window was real, opened when the last consuming PR merged, and closed with the flip. **Do this both-directions dry-run on every future wave** — it converts "same-day, trust me" into a measurement. Five things P3 proved that were not on its test list. A `constructed` dataset's builder must land in the **same** PR as the data, because `check_consumed_files.py` asserts the `builder:` path resolves. `builders/README.md`'s coverage table is a real coverage report and goes stale silently. The plain-git decision costs ~10 MB of packed history for 110 MB of working tree, since CSV compresses 5-22×. The **C0 → C1 → C2 ordering worked and proved less than it looks like** — the sync PR it was designed to defuse (QuantEcon/lecture-intro.zh-cn#293) touched zero data-read lines, zero `# i18n` markers and zero protected localisations, but nothing ever asked the model to rewrite those cells, so the markers remain unexercised, prompt-level protection and **the hand-diff is what protects a localisation**. And the translation sync is **`.md`-only**, so no hand-localised `_static` asset can be created, updated or repaired by it — every `data.ipynb` copy had to be repointed by hand in all four repos, filed upstream as QuantEcon/action-translation#271 -- [ ] **P4 — dynamic snapshot twin**: `UNRATE`, consumed today by 4 lectures across 3 repos via 2 access methods. Tests: the full dynamic template — manifest, four-stage builder, refresh-as-PR, canary catching an induced failure — plus the documented live-call ↔ snapshot switch mechanism +- [ ] **P4 — dynamic snapshot twin**: originally `UNRATE` alone; **reframed 2026-09-01** as the `business_cycle` set, because the lecture that needs a twin is excluded from `lecture-wasm` for want of one and a partial twin buys it nothing. Done so far: `business_cycle_data.csv` manifested and its builder retrofitted ([#109](https://github.com/QuantEcon/data-lectures/pull/109)); the refresh-as-PR and canary workflow ([#110](https://github.com/QuantEcon/data-lectures/pull/110)); the first real refresh ([#112](https://github.com/QuantEcon/data-lectures/pull/112)); the World Bank set extended to three tables and the FRED half landed as one composite monthly file on a shared `builders/_fred.py` library ([#114](https://github.com/QuantEcon/data-lectures/pull/114)). Remaining: the `lecture-wasm` adoption ([QuantEcon/lecture-wasm#70](https://github.com/QuantEcon/lecture-wasm/issues/70) — intro keeps its live calls as the lesson), the flip with `on_refresh: rebuild`, and a canary run catching an induced failure - [ ] Verify each migrated URL with a pyodide/JupyterLite fetch (CORS, meta#143) - [ ] Fold every validated decision into the draft `styleguide/datasets.md` (manual#108) as it is proven diff --git a/builders/README.md b/builders/README.md index ecbd38a..4aa79f8 100644 --- a/builders/README.md +++ b/builders/README.md @@ -27,7 +27,12 @@ upstream outage may fail a refresh, it must never break a lecture build. The architecture discussion is in [#14](https://github.com/QuantEcon/data-lectures/issues/14); the copy-able template is [`_template.py`](_template.py) (not a builder — the underscore -keeps it out of any manifest). +keeps it out of any manifest). Shared fetch code lives beside it under the +same convention: [`_fred.py`](_fred.py) is the `Fred` class every FRED +builder should use (`fred.series('UNRATE')`, `fred.frame([...])`), so a fetch +stage is a line and `validate()` is the only thing worth reading. One +builder per **source** for a lecture's data, writing a composite file where +the lecture reads the series together (decided 2026-09-01 on #26). A **dynamic snapshot's** builder additionally honours the refresh contract (`.github/workflows/refresh-snapshots.yml`, `scripts/snapshots.py`): @@ -59,7 +64,8 @@ re-fetched** — see `AGENTS.md`. | `fred_data.py` | `fred_data.csv` | committed — fetches six FRED series live over a pinned 1953-04..2024-12 window (yields and the recession dummy are stable history, unlike the BBH national-accounts snapshot). Reproduces its output byte for byte (2026-08-18) | | `hansen_singleton_1982_data.py` | `hansen_singleton_1982_data.csv` | committed — fetches FRED and the Ken French factors live. Reproduces its output byte for byte (2026-08-13) | | `hansen_singleton_1983_data.py` | `hansen_singleton_1983_data.csv` | committed — the same construction plus a T-bill leg, so its output is a strict superset of the 1982 file's. Reproduces its output byte for byte (2026-08-13) | -| `business_cycle.py` | `business_cycle_data.csv` (plus two dumps to `provenance/`) | committed — the repo's one **dynamic snapshot** (`cadence: annual`), retrofitted to the four-stage contract 2026-09-01. Fetches live WDI; does NOT reproduce its bytes and is not meant to — the World Bank revises the series (63 of 64 year columns moved between the 2025-02 vintage and 2026-09-01). validate() bounds the overlap window at 5 pp and prints the revision summary, which is the review surface for a refresh PR | +| `business_cycle.py` | `business_cycle_data.csv`, `unemployment_rate_annual.csv`, `private_credit_to_gdp.csv` (plus two dumps to `provenance/`) | committed — the World Bank half of the `business_cycle` lecture's data as three **dynamic snapshots** (`cadence: annual`), one builder writing a set. Fetches live WDI; does NOT reproduce its bytes and is not meant to — the World Bank revises the series (63 of 64 GDP-growth columns moved between the 2025-02 vintage and 2026-09-01). validate() places nulls (before an economy's first observation or in the newest two years, never inside a series), bounds each table's overlap window, and prints the summary — the review surface for a refresh PR. Validates all three before writing any | +| `business_cycle_fred.py` | `us_business_cycle_monthly.csv` | committed — the FRED half as one composite **dynamic snapshot** (`cadence: monthly`): six series on a monthly grid from 1919. Built on the shared `_fred.py` library; declares every structural null exactly (series starts, UMCSENT's sparse pre-1978 years, the 2025-10 shutdown hole) so a new hole fails the refresh | | `webscrape_forbes.ipynb` | `forbes-global2000.csv`, `forbes-billionaires.csv` | **committed-frozen** — an undocumented Forbes API, a spoofed user-agent and hardcoded GDPR consent cookies. Defects recorded in the two manifests rather than fixed | | `generating_mini.md` | `SCF_plus_mini.csv`, `SCF_plus_mini_no_weights.csv` | **committed-frozen** — its `to_csv` calls are commented out upstream and stay that way. As written it still fetches the `high_dim_data` URL; that URL is historical, and the input is now committed at `sources/SCF_plus.dta`. See `sources/README.md` | | `usa-gini-nwealth-tincome-lincome.ipynb` | `usa-gini-nwealth-tincome-lincome.csv` | **committed-frozen** — three independent reasons, any one sufficient: no validate stage; it raises under the pinned pandas 3 (`np.asarray` of a Series is read-only under copy-on-write, so `rd.shuffle` fails — the lecture got the `.copy()` fix in QuantEcon/lecture-python-intro#776, this notebook did not); and it is non-deterministic, so it cannot reproduce its own bytes. It is also the only builder here whose input is **another file in this repo** | diff --git a/builders/_fred.py b/builders/_fred.py new file mode 100644 index 0000000..d7b8498 --- /dev/null +++ b/builders/_fred.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Shared FRED fetch library for builders -- the `Fred` class. + +One place for the things every FRED read has to get right, so a builder's +fetch stage is a line or two and its validate() is the only thing worth +reading. Plain HTTP GET against `fredgraph.csv`, no `pandas_datareader` (the +decision on QuantEcon/data-lectures#26: a wrapper library in the builder +re-creates the single-point fragility Phase 5 exists to remove). + + from _fred import Fred + fred = Fred() + unrate = fred.series('UNRATE') # full history + yields = fred.frame(['GS1', 'GS10'], start='1953-04-01', end='2024-12-01') + tips = fred.series('DFII5', freq='Monthly', agg='avg') # daily -> monthly + +What it normalises: + +- the date column, which fredgraph titles `observation_date` today and + `DATE` in older exports -- every Series/DataFrame comes back with a + DatetimeIndex named `DATE`, so committed files keep the header the + lectures expect +- FRED's `.` for a missing observation -> NaN +- a User-Agent header, which fred.stlouisfed.org has been seen to require +- one series per request, aligned with an outer join in frame(), so a + series that starts later is simply empty before its first observation + +Not a builder: the leading underscore keeps it out of any manifest's +`builder:` field. Import it as `from _fred import Fred` (builders run with +their own directory as sys.path[0]). +""" +import io +import urllib.request + +import pandas as pd + +FREDGRAPH = 'https://fred.stlouisfed.org/graph/fredgraph.csv' + + +class Fred: + def __init__(self, user_agent='qeld-builder', timeout=60): + self.user_agent = user_agent + self.timeout = timeout + + def _get(self, params): + query = '&'.join(f'{k}={v}' for k, v in params.items() if v is not None) + request = urllib.request.Request(f'{FREDGRAPH}?{query}', + headers={'User-Agent': self.user_agent}) + with urllib.request.urlopen(request, timeout=self.timeout) as response: + return response.read() + + def series(self, sid, start=None, end=None, freq=None, agg=None): + """One FRED series as a pd.Series named `sid`, indexed by DATE. + + `start`/`end` are ISO dates (fredgraph's `cosd`/`coed`); `freq` and + `agg` request a server-side frequency change, e.g. freq='Monthly', + agg='avg' for the monthly mean of a daily series (fredgraph's + `fq`/`fam`).""" + payload = self._get({'id': sid, 'cosd': start, 'coed': end, 'fq': freq, 'fam': agg}) + frame = pd.read_csv(io.BytesIO(payload), index_col=0, parse_dates=True, na_values='.') + if frame.shape[1] != 1: + raise ValueError(f'{sid}: expected one value column, got {list(frame.columns)}') + s = frame.iloc[:, 0].rename(sid) + s.index.name = 'DATE' + return s + + def frame(self, sids, start=None, end=None): + """Several series on one DATE index (outer join, sorted).""" + out = pd.concat([self.series(sid, start, end) for sid in sids], axis=1).sort_index() + out.index.name = 'DATE' + return out diff --git a/builders/business_cycle.py b/builders/business_cycle.py index e2b7790..0607890 100644 --- a/builders/business_cycle.py +++ b/builders/business_cycle.py @@ -1,58 +1,44 @@ #!/usr/bin/env python3 """ -Builder for lectures/business_cycle_data.csv. - -Fetches annual real GDP growth (World Bank WDI series NY.GDP.MKTP.KD.ZG) for -five economies -- USA, ARG, GBR, GRC and JPN -- exactly as the intro -`business_cycle` lecture fetches it live with wbgapi, and writes it as one -wide CSV: one row per economy, one `YR` column per year from 1960. - -This is the repo's one DYNAMIC SNAPSHOT (`class: dynamic-snapshot`, -`cadence: annual` in the manifest). Unlike the frozen extracts beside it, the -World Bank revises this series continuously -- national-accounts rebasing -moves historical growth rates by up to about 1.5 percentage points -- so a -refresh is NOT expected to reproduce the committed bytes, and validate() does -not ask it to. Measured 2026-09-01: 63 of the 64 overlapping year columns had -at least one revised cell (236 of 320 cells; median change 0.0, 99th -percentile 1.0, maximum 1.5), and two new columns (YR2024, YR2025) had -appeared. That delta is recorded in the manifest's integrity.upstream block -and in the register at QuantEcon/data-lectures#39. - -What validate() DOES assert is the contract a consumer can rely on: the -shape, the column grid, the fixed set of economies, percent units, the one -structural null (YR1960, growth being undefined in the series' first year), -recency, and -- against the previously committed snapshot -- that no revision -exceeds MAX_REVISION percentage points and no previously populated cell has -gone empty. It prints the overlap-window summary on every run; that summary -is the review surface for a refresh PR (PLAN Phase 5). - -Two provenance dumps are written beside the data, to provenance/ (NOT -lectures/ -- they are not datasets; QuantEcon/data-lectures#13): the series -metadata, which is where the CC BY-4.0 licence the manifest cites is stated, -and the `wb.series.info(q='GDP growth')` listing the lecture teaches. Runs of -blank lines in the World Bank's text are collapsed to one (whitespace carries -no evidence, and the raw dump had runs ten newlines deep), so a refresh diff -of the dump shows what the source SAID, not how it was padded. - -Stages: fetch -> pre-process -> validate -> write. Writes only on validation -pass, and each write goes through a temp file and os.replace(), so neither a -failed refresh nor an interrupted one can leave anything but the last-good -snapshot in place. - -Usage: - python builders/business_cycle.py # refresh in place - python builders/business_cycle.py --out-dir D # dry run: write to D, - # still validating - # against lectures/ - python builders/business_cycle.py --summary-json S # also write the - # machine-readable - # run summary to S - -Exit codes (the contract .github/workflows/refresh-snapshots.yml relies on): - 0 wrote (or dry-ran) a validated snapshot - 1 fetch or other infrastructure failure -- retry, the data is not at fault - 2 ValidationError -- the fetched data broke the contract; needs a human - +Builder for the World Bank half of the `business_cycle` lecture's data -- +three annual tables from WDI, each in the wide layout wbgapi emits (one row +per economy, ISO3 code as the index, country name as `Country`, one `YR` +column per year from 1960): + + business_cycle_data.csv NY.GDP.MKTP.KD.ZG real GDP growth, %, for + the nine economies the lecture plots (the + union of its two selections) + unemployment_rate_annual.csv SL.UEM.TOTL.NE.ZS unemployment, % of labour + force (national estimate), USA FRA GBR JPN + private_credit_to_gdp.csv FS.AST.PRVT.GD.ZS domestic credit to the + private sector, % of GDP, GBR + +One builder, three files: the "builder writes a set" precedent +(builders/README.md). The two new filenames are PROVISIONAL pending the naming +policy (QuantEcon/data-lectures#113); they are free to change while no lecture +reads them. + +These are DYNAMIC SNAPSHOTS (`cadence: annual`). The World Bank revises this +data continuously -- national-accounts rebasing moved GDP growth by up to 1.5 +percentage points between the 2025-02 and 2026-09 vintages -- so a refresh is +NOT expected to reproduce the committed bytes and validate() does not ask it +to. What it asserts is the contract a consumer can rely on: the grid, the +fixed economy set, units, structurally-placed nulls, recency, and a bounded +overlap window against the committed snapshot, printed as the review surface +for the refresh PR. + +Nulls: WDI series start at different years per economy (UK and French +unemployment begin in 1971 and 1970), GDP growth is undefined in 1960 for +everyone, and the newest year may not be published yet for every series. So +the rule is structural, not a count: a null is allowed only BEFORE an +economy's first observation or in the newest MAX_TRAILING_YEARS, never inside +the series. A gap opening mid-series fails the refresh. + +Two provenance dumps (the GDP series' metadata, where the CC BY-4.0 licence +is stated, and the `wb.series.info` listing the lecture teaches) go to +provenance/, with runs of blank lines collapsed. Stages: fetch -> pre-process +-> validate -> write; --out-dir dry-runs; --summary-json writes one summary +PER FILE as a JSON list. Exit 2 on ValidationError, 1 on a fetch failure. Requires pandas and wbgapi (requirements.txt). """ import argparse @@ -70,35 +56,34 @@ PUBLISHED_DIR = os.path.join(REPO_ROOT, 'lectures') PROVENANCE_DIR = os.path.join(REPO_ROOT, 'provenance') -OUT_FILE = 'business_cycle_data.csv' METADATA_FILE = 'business_cycle_metadata.md' INFO_FILE = 'business_cycle_info.md' - -SERIES = 'NY.GDP.MKTP.KD.ZG' -# The five economies the lecture plots, in the order it names them. wbgapi -# returns rows in its own order (JPN, GRC, GBR, ARG, USA on every run so far); -# that order is kept as returned so a refresh diff is values-only. -ECONOMIES = ['USA', 'ARG', 'GBR', 'GRC', 'JPN'] FIRST_YEAR = 1960 YEAR_COL = re.compile(r'^YR(\d{4})$') - -# Overlap-window policy for a revised aggregate. The World Bank's routine -# revisions to this series have measured at most 1.5 pp (2026-09-01, 2025-02 -# vintage vs live); a change past this bound is a rebasing, a units switch or -# an upstream defect, and wants a human before it ships. -MAX_REVISION = 5.0 # percentage points, any single cell -# Percent units: a fetch that came back as a ratio (0.02 for 2%) or as an -# index level would pass every shape check and rescale the lecture's figure. -MIN_ABS_MAX, MAX_ABS = 1.0, 50.0 -# A refresh whose newest year is older than this many years behind today -# means the fetch returned a stale or truncated panel. MAX_STALENESS_YEARS = 2 +MAX_TRAILING_YEARS = 2 # the newest years may be unpublished for a series + +# One entry per published file. `economies` is the lecture's selection (the +# union of every call that reads the series); `band` is the unit sanity check +# (percent / percent / percent of GDP); `max_revision` bounds the overlap +# window in the series' own units (GDP growth measured at 1.5 pp routine; +# the credit ratio is rebased in larger steps). +TABLES = [ + {'file': 'business_cycle_data.csv', 'series': 'NY.GDP.MKTP.KD.ZG', + 'economies': ['USA', 'ARG', 'GBR', 'GRC', 'JPN', 'CHN', 'DEU', 'BRA', 'MEX'], + 'band': (-50, 50), 'min_abs_max': 1.0, 'max_revision': 5.0, + 'first_year_null': True}, # growth is undefined in the series' first year + {'file': 'unemployment_rate_annual.csv', 'series': 'SL.UEM.TOTL.NE.ZS', + 'economies': ['USA', 'FRA', 'GBR', 'JPN'], + 'band': (0, 60), 'min_abs_max': 1.0, 'max_revision': 3.0, 'first_year_null': False}, + {'file': 'private_credit_to_gdp.csv', 'series': 'FS.AST.PRVT.GD.ZS', + 'economies': ['GBR'], + 'band': (0, 400), 'min_abs_max': 1.0, 'max_revision': 25.0, 'first_year_null': False}, +] class ValidationError(Exception): - """The fetched data broke the published contract -- distinct from a fetch - failure, so CI can tell 'the data is wrong' (a human) from 'the network - was down' (a retry). Exit code 2.""" + """The fetched data broke the published contract -- exit code 2.""" def _check(condition, message): @@ -107,22 +92,17 @@ def _check(condition, message): def _tidy(text): - """Collapse runs of blank lines in an upstream text dump to one.""" return re.sub(r'\n{3,}', '\n\n', text) def fetch(): - """Live WDI, exactly the lecture's own three calls.""" - frame = wb.data.DataFrame(SERIES, ECONOMIES, labels=True) - metadata = _tidy(str(wb.series.metadata.get(SERIES))) + frames = {t['file']: wb.data.DataFrame(t['series'], t['economies'], labels=True) for t in TABLES} + metadata = _tidy(str(wb.series.metadata.get(TABLES[0]['series']))) info = _tidy(str(wb.series.info(q='GDP growth'))) - return frame, metadata, info + return frames, metadata, info def pre_process(frame): - # wbgapi's index is the ISO3 code, named `economy`; `Country` is the label - # column `labels=True` adds. Keep the layout the lecture (and the - # committed file) uses: Country first, then the year columns ascending. frame = frame.copy() frame.index.name = 'economy' years = sorted(c for c in frame.columns if YEAR_COL.match(c)) @@ -133,77 +113,74 @@ def _years(frame): return [int(YEAR_COL.match(c).group(1)) for c in frame.columns if YEAR_COL.match(c)] -def validate(frame, previous=None): - """Refuse to write anything that is not the shape we expect. Returns the - run summary (the refresh PR's body and the manifest stamp are built from - it), raising ValidationError on the first broken invariant.""" - # Grid: Country, then YR..YR with no gap. - _check(list(frame.columns[:1]) == ['Country'], f'first columns {list(frame.columns[:3])}') +def validate(table, frame, previous=None): + name = table['file'] + _check(list(frame.columns[:1]) == ['Country'], f'{name}: first columns {list(frame.columns[:3])}') years = _years(frame) - _check(len(years) == len(frame.columns) - 1, 'non-year column present') - _check(years[0] == FIRST_YEAR, f'first year is {years[0]}, not {FIRST_YEAR}') - _check(years == list(range(FIRST_YEAR, years[-1] + 1)), 'gap in the year grid') + _check(len(years) == len(frame.columns) - 1, f'{name}: non-year column present') + _check(years[0] == FIRST_YEAR, f'{name}: first year {years[0]}') + _check(years == list(range(FIRST_YEAR, years[-1] + 1)), f'{name}: gap in the year grid') year_cols = [f'YR{y}' for y in years] - - # Economies: exactly the five, one row each. - _check(frame.index.name == 'economy', f'index is {frame.index.name!r}') - _check(sorted(frame.index) == sorted(ECONOMIES), f'economies {sorted(frame.index)}') - _check(frame['Country'].notnull().all(), 'a Country label is missing') - - # Dtypes and units. + _check(frame.index.name == 'economy', f'{name}: index is {frame.index.name!r}') + _check(sorted(frame.index) == sorted(table['economies']), f'{name}: economies {sorted(frame.index)}') + _check(frame['Country'].notnull().all(), f'{name}: a Country label is missing') values = frame[year_cols] - _check(all(pd.api.types.is_float_dtype(values[c]) for c in year_cols), 'non-float year column') - _check(values.abs().max().max() >= MIN_ABS_MAX, 'values look like ratios, not percent') - _check(values.abs().max().max() <= MAX_ABS, 'growth rate out of band') - - # The one structural null: growth is undefined in the series' first year. - nulls = values.isnull().sum() - _check(dict(nulls[nulls > 0]) == {f'YR{FIRST_YEAR}': len(ECONOMIES)}, - f'unexpected nulls {dict(nulls[nulls > 0])}') - - # Recency. - _check(years[-1] >= dt.date.today().year - MAX_STALENESS_YEARS, f'newest year is {years[-1]}') + _check(all(pd.api.types.is_float_dtype(values[c]) for c in year_cols), f'{name}: non-float year column') + _check(values.abs().max().max() >= table['min_abs_max'], f'{name}: values look like ratios') + lo, hi = table['band'] + _check(values.stack().between(lo, hi).all(), f'{name}: value out of band [{lo}, {hi}]') + _check(years[-1] >= dt.date.today().year - MAX_STALENESS_YEARS, f'{name}: newest year is {years[-1]}') + + # Nulls: only before an economy's first observation, or in the newest + # MAX_TRAILING_YEARS; never inside the series. GDP growth additionally has + # its first year empty for everyone. + trailing = set(year_cols[-MAX_TRAILING_YEARS:]) + for econ in frame.index: + row = values.loc[econ] + first = row.first_valid_index() + _check(first is not None, f'{name}: {econ} has no data at all') + inner = row.loc[first:] + bad = [c for c in inner.index if pd.isnull(inner[c]) and c not in trailing] + _check(not bad, f'{name}: {econ} has a gap inside its series at {bad[:3]}') + if table['first_year_null']: + _check(pd.isnull(row[year_cols[0]]), f'{name}: {econ} has a value in {year_cols[0]}') summary = { - 'dataset': OUT_FILE, + 'dataset': name, 'builder': os.path.relpath(os.path.abspath(__file__), REPO_ROOT), 'rows': int(frame.shape[0]), 'columns': int(frame.shape[1]), 'date_range': {'start': years[0], 'end': years[-1]}, 'overlap': None, } - - # Overlap window against the last-good snapshot: revisions are expected, - # bounded, and reported; a lost observation or a rescale is not. if previous is not None: prev_years = [f'YR{y}' for y in _years(previous)] - _check(set(prev_years) <= set(year_cols), 'a year column disappeared') - _check(sorted(previous.index) == sorted(frame.index), 'the economy set changed') - old = previous.loc[frame.index, prev_years] - new = frame.loc[frame.index, prev_years] - _check(not (old.notnull() & new.isnull()).any().any(), 'a populated cell went empty') + _check(set(prev_years) <= set(year_cols), f'{name}: a year column disappeared') + common = [e for e in previous.index if e in frame.index] + _check(common, f'{name}: no economy in common with the previous snapshot') + old = previous.loc[common, prev_years] + new = frame.loc[common, prev_years] + _check(not (old.notnull() & new.isnull()).any().any(), f'{name}: a populated cell went empty') diff = (old - new).abs() changed = int((diff > 1e-9).sum().sum()) - worst = float(diff.max().max()) - new_cols = sorted(set(year_cols) - set(prev_years)) + worst = float(diff.max().max()) if diff.notnull().any().any() else 0.0 summary['overlap'] = { 'window': f'{prev_years[0]}..{prev_years[-1]}', 'previous_end': _years(previous)[-1], - 'cells_total': int(diff.size), + 'cells_total': int(old.notnull().sum().sum()), 'cells_revised': changed, 'max_abs_change': round(worst, 4), - 'new_columns': new_cols, + 'new_columns': sorted(set(year_cols) - set(prev_years)), + 'new_economies': sorted(set(frame.index) - set(previous.index)), } - print(f'overlap window {prev_years[0]}..{prev_years[-1]}: ' - f'{changed} of {diff.size} cells revised, max |change| {worst:.3f} pp; ' - f'new columns: {new_cols or "none"}') - _check(worst <= MAX_REVISION, f'revision of {worst:.3f} pp exceeds {MAX_REVISION}') + print(f'{name}: overlap {prev_years[0]}..{prev_years[-1]} over {common}: {changed} cells revised, ' + f'max |change| {worst:.3f}; new columns {summary["overlap"]["new_columns"] or "none"}; ' + f'new economies {summary["overlap"]["new_economies"] or "none"}') + _check(worst <= table['max_revision'], f'{name}: revision of {worst:.3f} exceeds {table["max_revision"]}') return summary def _atomic_write(path, text): - """Write via a same-directory temp file and os.replace(), so an interrupted - run cannot leave a truncated file where the last-good snapshot was.""" tmp = path + '.tmp' with open(tmp, 'w') as f: f.write(text) @@ -213,33 +190,35 @@ def _atomic_write(path, text): def run(out_dir=None, summary_json=None): data_dir = out_dir or PUBLISHED_DIR prov_dir = out_dir or PROVENANCE_DIR - previous_path = os.path.join(PUBLISHED_DIR, OUT_FILE) - previous = (pd.read_csv(previous_path, index_col=0) - if os.path.exists(previous_path) else None) - - frame, metadata, info = fetch() - frame = pre_process(frame) - summary = validate(frame, previous) + frames, metadata, info = fetch() + summaries, outputs = [], {} + for table in TABLES: + previous_path = os.path.join(PUBLISHED_DIR, table['file']) + previous = pd.read_csv(previous_path, index_col=0) if os.path.exists(previous_path) else None + frame = pre_process(frames[table['file']]) + summaries.append(validate(table, frame, previous)) + outputs[table['file']] = frame + # Validate everything, then write everything: a failure in the third table + # must not leave the first two refreshed and the set out of step. if summary_json: - _atomic_write(summary_json, json.dumps(summary, indent=1) + '\n') - + _atomic_write(summary_json, json.dumps(summaries, indent=1) + '\n') os.makedirs(data_dir, exist_ok=True) os.makedirs(prov_dir, exist_ok=True) - _atomic_write(os.path.join(data_dir, OUT_FILE), frame.to_csv()) + for name, frame in outputs.items(): + _atomic_write(os.path.join(data_dir, name), frame.to_csv()) + years = _years(frame) + print(f'wrote {name}: {frame.shape[0]} economies x {len(years)} years ({years[0]} .. {years[-1]}) -> {data_dir}') _atomic_write(os.path.join(prov_dir, METADATA_FILE), metadata) _atomic_write(os.path.join(prov_dir, INFO_FILE), info) - years = _years(frame) - print(f'wrote {OUT_FILE}: {frame.shape[0]} economies x {len(years)} years ' - f'({years[0]} .. {years[-1]}) -> {data_dir}') if __name__ == '__main__': ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) ap.add_argument('--out-dir', help='write outputs here instead of lectures/ and provenance/') - ap.add_argument('--summary-json', help='also write the run summary (JSON) here') + ap.add_argument('--summary-json', help='also write the run summaries (a JSON list) here') args = ap.parse_args() try: run(args.out_dir, args.summary_json) except ValidationError as exc: - print(f'::error::{OUT_FILE}: validation failed -- {exc}', file=sys.stderr) + print(f'::error::business_cycle: validation failed -- {exc}', file=sys.stderr) sys.exit(2) diff --git a/builders/business_cycle_fred.py b/builders/business_cycle_fred.py new file mode 100644 index 0000000..cdac113 --- /dev/null +++ b/builders/business_cycle_fred.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +""" +Builder for lectures/us_business_cycle_monthly.csv -- the FRED half of the +`business_cycle` lecture's data, as one composite monthly file. + +Six FRED series on one monthly DATE index from 1919-01 (the earliest month +the lecture asks for, INDPRO's start) to the newest month FRED carries: + + UNRATE civilian unemployment rate, %, monthly from 1948-01 + USREC NBER recession indicator, 0/1, monthly (from 1854) + UMCSENT Michigan consumer sentiment, index, irregular from + 1952-11 and MONTHLY only from 1978-01 -- the lecture + reads it from 1978, and validate() treats the earlier + gaps as structural + CPILFESL CPI less food and energy, index 1982-84=100, from 1957-01 + INDPRO industrial production index, 2017=100, from 1919-01 + M0892AUSM156SNBR unemployment rate 1929-04..1942-06 (NBER Macrohistory), + the historical series the lecture splices before UNRATE + +Why one file rather than six: the lecture reads them together, and a single +URL with the lecture's own variable names as columns is what keeps the wasm +edition readable (QuantEcon/lecture-wasm#70). The fetch is `builders/_fred.py`; +this file is the contract. + +Nulls are structural and declared, not tolerated: each series is empty before +its first observation; UMCSENT is sparse before 1978-01; the historical series +is empty after 1942-06; and UNRATE and CPILFESL share ONE hole, 2025-10, the +month the 2025 federal shutdown stopped BLS publishing. validate() asserts +exactly those and no others, so a new hole -- another shutdown, a FRED outage +served as `.` -- fails the refresh for a human to look at rather than shipping. + +Revisions: CPILFESL and INDPRO are revised (seasonal factors yearly, INDPRO's +annual benchmark), UNRATE occasionally, USREC and the historical series never. +validate() bounds each series' revisions in the overlap window and prints the +summary -- the review surface for the refresh PR. + +Stages: fetch -> pre-process -> validate -> write. --out-dir dry-runs, +--summary-json writes the run summary. Exit 2 on ValidationError, 1 on a +fetch failure. Requires pandas. +""" +import argparse +import datetime as dt +import json +import os +import sys + +import pandas as pd + +from _fred import Fred + +CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(CURRENT_FILE_DIR) +PUBLISHED_DIR = os.path.join(REPO_ROOT, 'lectures') + +OUT_FILE = 'us_business_cycle_monthly.csv' +START = '1919-01-01' +COLUMNS = ['UNRATE', 'USREC', 'UMCSENT', 'CPILFESL', 'INDPRO', 'M0892AUSM156SNBR'] + +# First observation of each series, as published; a series whose start moves +# is a different series. +FIRST_OBS = {'UNRATE': '1948-01-01', 'USREC': START, 'UMCSENT': '1952-11-01', + 'CPILFESL': '1957-01-01', 'INDPRO': START, 'M0892AUSM156SNBR': '1929-04-01'} +# Where each series becomes gap-free monthly. UMCSENT is quarterly/irregular +# before 1978; the historical unemployment series ends in 1942-06. +MONTHLY_FROM = {**{c: FIRST_OBS[c] for c in COLUMNS}, 'UMCSENT': '1978-01-01'} +LAST_OBS = {'M0892AUSM156SNBR': '1942-06-01'} +# Holes inside a series' monthly span that are known and accepted. +KNOWN_HOLES = {'UNRATE': ['2025-10-01'], 'CPILFESL': ['2025-10-01']} +# Value bands: percent, 0/1, index levels. +BANDS = {'UNRATE': (0, 30), 'USREC': (0, 1), 'UMCSENT': (20, 150), + 'CPILFESL': (20, 1000), 'INDPRO': (1, 300), 'M0892AUSM156SNBR': (0, 40)} +# Overlap-window bound per series, in the series' own units. +MAX_REVISION = {'UNRATE': 0.5, 'USREC': 0, 'UMCSENT': 5.0, + 'CPILFESL': 3.0, 'INDPRO': 5.0, 'M0892AUSM156SNBR': 0} +MAX_STALENESS_MONTHS = 3 + + +class ValidationError(Exception): + """The fetched data broke the published contract -- exit code 2.""" + + +def _check(condition, message): + if not condition: + raise ValidationError(message) + + +def fetch(): + return Fred().frame(COLUMNS, start=START) + + +def pre_process(frame): + frame = frame.loc[START:].copy() + frame.index.name = 'DATE' + frame['USREC'] = frame['USREC'].astype('int64') + return frame[COLUMNS] + + +def _months(index): + return pd.Series(index.year * 12 + index.month, index=index) + + +def validate(frame, previous=None): + _check(list(frame.columns) == COLUMNS, f'columns {list(frame.columns)}') + _check(frame.index.name == 'DATE', f'index is {frame.index.name!r}') + _check(frame.index[0] == pd.Timestamp(START), f'starts {frame.index[0].date()}') + _check(frame.index.is_monotonic_increasing and (frame.index.day == 1).all(), 'not first-of-month') + _check((_months(frame.index).diff().dropna() == 1).all(), 'gap in the monthly grid') + today = dt.date.today() + age = (today.year * 12 + today.month) - (frame.index[-1].year * 12 + frame.index[-1].month) + _check(age <= MAX_STALENESS_MONTHS, f'newest month is {frame.index[-1].date()}') + + for col in COLUMNS: + s = frame[col] + first = pd.Timestamp(FIRST_OBS[col]) + _check(s.loc[:first - pd.offsets.MonthBegin(1)].isnull().all() if first > frame.index[0] else True, + f'{col}: observed before its first observation {first.date()}') + _check(s.first_valid_index() == first, f'{col}: first observation is {s.first_valid_index()}, not {first.date()}') + monthly_from = pd.Timestamp(MONTHLY_FROM[col]) + last = pd.Timestamp(LAST_OBS.get(col, frame.index[-1])) + span = s.loc[monthly_from:last] + holes = [str(d.date()) for d in span[span.isnull()].index] + _check(holes == KNOWN_HOLES.get(col, []), f'{col}: holes {holes} != known {KNOWN_HOLES.get(col, [])}') + if col in LAST_OBS: + _check(s.loc[last + pd.offsets.MonthBegin(1):].isnull().all(), f'{col}: observed after {last.date()}') + lo, hi = BANDS[col] + _check(s.dropna().between(lo, hi).all(), f'{col}: out of band [{lo}, {hi}]') + _check(set(frame['USREC'].unique()) <= {0, 1}, 'USREC not 0/1') + + summary = { + 'dataset': OUT_FILE, + 'builder': os.path.relpath(os.path.abspath(__file__), REPO_ROOT), + 'rows': int(frame.shape[0]), + 'columns': int(frame.shape[1]), + 'date_range': {'start': str(frame.index[0].date()), 'end': str(frame.index[-1].date())}, + 'overlap': None, + } + if previous is not None: + _check(list(previous.columns) == COLUMNS, 'previous snapshot has different columns') + common = previous.index.intersection(frame.index) + old, new = previous.loc[common], frame.loc[common] + _check(not (old.notnull() & new.isnull()).any().any(), 'a populated cell went empty') + diff = (old - new).abs() + worst = {c: float(diff[c].max()) if diff[c].notnull().any() else 0.0 for c in COLUMNS} + changed = int((diff > 1e-9).sum().sum()) + summary['overlap'] = { + 'window': f'{common[0].date()}..{common[-1].date()}', + 'previous_end': str(previous.index[-1].date()), + 'cells_total': int(old.notnull().sum().sum()), + 'cells_revised': changed, + 'max_abs_change': round(max(worst.values()), 4), + 'max_abs_change_by_series': {c: round(v, 4) for c, v in worst.items()}, + 'new_columns': [], + } + print(f'overlap window {common[0].date()}..{common[-1].date()}: {changed} cells revised; ' + f'max |change| by series {summary["overlap"]["max_abs_change_by_series"]}') + for c in COLUMNS: + _check(worst[c] <= MAX_REVISION[c], f'{c}: revision {worst[c]:.4f} exceeds {MAX_REVISION[c]}') + return summary + + +def _atomic_write(path, text): + tmp = path + '.tmp' + with open(tmp, 'w') as f: + f.write(text) + os.replace(tmp, path) + + +def run(out_dir=None, summary_json=None): + data_dir = out_dir or PUBLISHED_DIR + previous_path = os.path.join(PUBLISHED_DIR, OUT_FILE) + previous = (pd.read_csv(previous_path, index_col=0, parse_dates=True) + if os.path.exists(previous_path) else None) + frame = pre_process(fetch()) + summary = validate(frame, previous) + if summary_json: + _atomic_write(summary_json, json.dumps(summary, indent=1) + '\n') + os.makedirs(data_dir, exist_ok=True) + _atomic_write(os.path.join(data_dir, OUT_FILE), frame.to_csv()) + print(f'wrote {OUT_FILE}: {frame.shape[0]} months x {frame.shape[1]} series ' + f'({frame.index[0].date()} .. {frame.index[-1].date()}) -> {data_dir}') + + +if __name__ == '__main__': + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('--out-dir') + ap.add_argument('--summary-json') + args = ap.parse_args() + try: + run(args.out_dir, args.summary_json) + except ValidationError as exc: + print(f'::error::{OUT_FILE}: validation failed -- {exc}', file=sys.stderr) + sys.exit(2) diff --git a/lectures/business_cycle_data.csv b/lectures/business_cycle_data.csv index f2ba36c..29250ca 100644 --- a/lectures/business_cycle_data.csv +++ b/lectures/business_cycle_data.csv @@ -1,4 +1,8 @@ economy,Country,YR1960,YR1961,YR1962,YR1963,YR1964,YR1965,YR1966,YR1967,YR1968,YR1969,YR1970,YR1971,YR1972,YR1973,YR1974,YR1975,YR1976,YR1977,YR1978,YR1979,YR1980,YR1981,YR1982,YR1983,YR1984,YR1985,YR1986,YR1987,YR1988,YR1989,YR1990,YR1991,YR1992,YR1993,YR1994,YR1995,YR1996,YR1997,YR1998,YR1999,YR2000,YR2001,YR2002,YR2003,YR2004,YR2005,YR2006,YR2007,YR2008,YR2009,YR2010,YR2011,YR2012,YR2013,YR2014,YR2015,YR2016,YR2017,YR2018,YR2019,YR2020,YR2021,YR2022,YR2023,YR2024,YR2025 +MEX,Mexico,,5.00000000072309,4.66441467253127,8.10688692761114,11.9054807657866,7.09999999747173,6.09613930256681,5.85492487897324,9.42327881831926,3.41862002349839,6.50248403469675,3.76246768495716,8.22880731245257,7.86111986063614,5.77682723020618,5.74448504862428,4.41744413626853,3.39063970656844,8.95694232786306,9.69817013704461,8.75662242586424,9.58584115454994,-0.0495863656386035,-4.62412795987231,3.51341334932273,1.91921578613494,-3.92995990782299,2.06336322631584,1.21826122938586,3.62544873456009,5.25004329944582,3.97590497138043,3.56861354412243,2.86699945461075,4.39412788764169,-5.91029962358286,6.21829715869302,7.19887615284711,6.18507893082885,2.75503039300564,5.02928399378261,-0.450845253215945,-0.236588001922641,1.18554486709559,3.56544102622887,2.11324711473297,4.80501351664901,2.0778639395329,0.943331872912935,-6.29525056697287,4.9713345832195,3.44404505297089,3.55321076010608,0.852101550164704,2.50376350783593,2.70232343024026,1.7724932254128,1.87172854648381,1.97208211101069,-0.392690513290532,-8.35403459379062,6.04848347173572,3.70975708309098,3.1067964131456,1.35057631384696,0.561683456144578 +BRA,Brazil,,8.60000000000014,6.59999999999992,0.600000000000065,3.39999999999985,2.40000000000001,6.7000000000001,4.2,9.79999999999998,9.49999999999993,10.4,11.3429219931908,11.9403481162508,13.9687217796782,8.15393868457201,5.16664908406321,10.2571295347867,4.93432806978961,4.96989768924738,6.7595601220408,9.19999999999976,-4.24999999999956,0.829999999999529,-2.92999999999996,5.39999999999988,7.85000000000042,7.48999999999982,3.52999999999983,-0.059999999999917,3.16000000000032,-4.35000000000022,1.03218958549303,-0.54407204977727,4.9246900037071,5.85287036219013,4.22379363350413,2.20886405039022,3.39484598630671,0.338097901207306,0.467937566477133,4.38794944745618,1.38989640095637,3.05346185909312,1.14082899824888,5.75996463670954,3.20213206130431,3.96198871122617,6.06987060678338,5.09419544658736,-0.125812002161169,7.52822581815364,3.97442307944702,1.92117598576537,3.00482266944432,0.503955740242247,-3.54576339269425,-3.27591690782192,1.32286905404399,1.783666761634,1.22077782360842,-3.2767587964736,4.76260437908608,3.01669435393015,3.24165532906981,3.41931516501906,2.2857464902475 +DEU,Germany,,4.29843950975808,4.6234710102125,2.73529643602564,6.63946989651858,5.24416402432635,2.81218130587229,-0.330256452713016,5.66877310370654,7.41810273162662,5.10314457629295,3.13269976979846,4.30034129247663,4.77748690945469,0.890068699312494,-0.866738936529003,4.94925843456397,3.34721808033223,3.00849288842491,4.15036333786591,1.40882864441943,0.529240541969116,-0.394840735634787,1.57241014192158,2.82294783585535,2.32793520067574,2.28733925852798,1.4021515883601,3.70723567991882,3.89655170553733,5.25500610110028,5.10826150190911,2.01361340926105,-0.973033082340891,2.59685569693275,1.50499384201356,1.0378755674358,1.85432238398302,2.09561231172233,2.12957016297766,2.8765231247659,1.63614170690956,-0.228255715793637,-0.529801394291297,1.16208684507278,0.885485292551564,3.86668241523209,2.8891174138025,0.887902330743628,-5.54455451920674,4.13463778522758,3.75796884489337,0.463512036709204,0.396995583779145,2.18018602168017,1.66300602334775,2.22222216259371,2.797906720551,1.13569604599849,0.977734697038784,-4.13191443239947,3.90999994204108,1.8092581243061,-0.869647578573321,-0.495851897260366,0.239578342117881 +CHN,China,,-27.2700000038883,-5.57999999019671,10.2999999963989,18.1799999938203,16.9500000038699,10.6499999984174,-5.77000000405148,-4.09999998647768,16.9399999960609,19.2999999967022,7.05999999853672,3.81000000021984,7.76000000084325,2.31000000294736,8.71999999747992,-1.57000000165326,7.57000000138403,11.3269542129809,7.59480175999998,7.84141706847441,5.11486309071954,9.02434347373044,10.7718383060041,15.1921349118079,13.4284820016509,8.9437463904392,11.6552658383921,11.2209115803492,4.20785693658392,3.9225195692421,9.37563237902144,14.2976226152779,13.9261028311995,13.0827122987906,11.0154136659719,9.97328781228394,9.29759460595947,7.92354535108497,7.74123827728161,8.57357328926101,8.32404695347056,9.23604870383861,10.1138119025242,10.1404291872206,11.4535130030645,12.6752842281595,14.1507710506103,9.66877608164454,9.40610014336416,10.5916903885704,9.46205248298186,7.8578201481217,7.77842954757257,7.46100735778161,6.98162255562367,6.7736886867727,6.89123916636191,6.757806142324,6.06737806333672,2.33950385203023,8.57038414981781,3.13385438394069,5.41600334492955,4.95830378788091,4.95994886240992 JPN,Japan,,12.043536357022,8.90897303614973,8.47364243635415,11.676708250654,5.81970783966972,10.6385615902031,11.0821423327248,12.8824681707163,12.4778945420024,0.399050611065093,4.69899204214205,8.41354725497627,8.03259997458245,-1.22523982750916,3.09157591623077,3.97498409113672,4.39033795009327,5.27194150317187,5.4840418324284,2.81759120738283,4.20933646523171,3.31245673740017,3.52304498510377,4.50199481622435,5.23338094687848,3.32652561623384,4.73066586040017,6.78502010955764,4.85803768615389,4.89271306574565,3.41749676183089,0.848069581218368,-0.517919847117952,0.99306636322622,2.34205546196849,3.07546731409349,1.36297109407448,-1.77264091037956,-0.347600398675738,2.97832688768516,0.284608556379041,0.0849888050363319,1.38010935830795,2.11294823723127,1.8482813374834,1.54630843078741,1.72837912865577,-1.16662209355141,-5.9359411383964,4.13645589222239,-0.20035785332982,1.64893452981225,1.99727653399265,0.905915486065865,1.79997735783475,0.704986495111257,1.62345062245441,0.834143845965585,-0.308496921143259,-4.28327942675573,3.56419531474475,1.33164750653741,0.720785304749811,-0.240084236041696,1.19311288444599 GRC,Greece,,13.2038386052462,0.364812289877747,11.8448666655957,9.40967732368279,10.7680109720411,6.49450138060494,5.66948643356486,7.20371768036424,11.5636684082563,8.93058020025967,7.84117657760304,10.1601515908652,8.09237907118801,-6.43824057014004,6.36680794602181,6.85189869571943,2.94100090620763,7.24686397906468,3.28207958853159,0.67713140669585,-1.55372126905196,-1.13264739051145,-1.07862273931177,2.01058093704644,2.50955640237393,0.517660006338886,-2.25886322369171,4.28786172851761,3.79999995287797,0.0,3.09999970633838,0.700000337119789,-1.60000002186294,1.99999984724822,2.09971953235275,1.38247057325857,3.60288326516122,3.43693579661333,2.65878562321184,4.13782699405849,4.65041459442163,4.68313727063017,5.79682274294369,5.37786634595851,1.18350635493289,6.44339762487436,3.50686976639459,0.0574740238200775,-4.11927606715011,-5.6937412158003,-9.87677875300243,-8.33113421099992,-2.27211699888024,0.792225060962835,-0.228301892172851,-0.0317947678412054,1.47312472702352,2.06467207892409,2.27718065334808,-9.19623149726978,8.65449786002131,5.52198580589408,2.13591131996631,2.0865743380603,2.07306882481713 GBR,United Kingdom,,2.70131421328257,1.0986962362898,4.85954520244742,5.59481105874345,2.13033308666921,1.56744960482462,2.77573771112216,5.47269310050385,1.93913775370176,2.70858148856038,2.00453760135888,3.62637105743457,7.31352424220155,-1.67969792645223,-0.676870194049812,2.80465599545295,2.36335094409922,3.40337382855367,2.74677219709527,-2.17910220206564,-1.3234581733779,2.09306886432368,3.62434641923983,2.67092491032817,3.59938143817158,4.01232987098167,4.56208082572655,5.0321018040183,2.28144671930745,0.779266119883417,-1.29243974751705,1.29472849604493,3.49048744653493,4.95341092800692,3.53339096338078,2.61491606949487,4.8799853038024,3.24631482881618,2.94445869854798,4.52438074810904,2.39102229602824,1.70902736258567,3.21851612669614,2.32601853325993,2.7906547174688,2.18982132415704,2.93091133582351,-0.0494116727402769,-4.58740819226323,2.26054460931586,0.851437739020326,1.52826034721852,1.71846961308464,3.16017351468152,2.14040560033371,2.20652020635357,3.02322219840339,1.55133095078786,1.25629899760824,-10.047896637362,8.5431118453255,5.14970388522993,0.271650048355255,1.08027716858781,1.38844207858726 diff --git a/lectures/business_cycle_data.csv.yml b/lectures/business_cycle_data.csv.yml index 2943445..9772a64 100644 --- a/lectures/business_cycle_data.csv.yml +++ b/lectures/business_cycle_data.csv.yml @@ -12,16 +12,17 @@ # vintage under a new name. filename: business_cycle_data.csv -title: World Bank GDP growth (annual %) — USA, ARG, GBR, GRC, JPN, 1960 onward +title: World Bank GDP growth (annual %) — nine economies, 1960 onward description: > - Annual real GDP growth (percent, constant local currency) for the United - States, Argentina, the United Kingdom, Greece and Japan, extracted from World - Bank WDI series NY.GDP.MKTP.KD.ZG in the wide layout wbgapi emits: one row - per economy (ISO3 code as the index, country name as `Country`), one - `YR` column per year from 1960 to the newest year the source carries - (`schema.date_range.end`). It is the snapshot twin of the live - `wb.data.DataFrame` call the intro `business_cycle` lecture makes; the call - is reproduced verbatim by the builder. + Annual real GDP growth (percent, constant local currency) for the nine + economies the intro `business_cycle` lecture plots — USA, ARG, GBR, GRC, JPN + in its opening section and CHN, USA, DEU, BRA, ARG, GBR, JPN, MEX in its + developed/developing comparison — extracted from World Bank WDI series + NY.GDP.MKTP.KD.ZG in the wide layout wbgapi emits: one row per economy + (ISO3 code as the index, country name as `Country`), one `YR` column + per year from 1960 to the newest year the source carries + (`schema.date_range.end`). The snapshot twin of the lecture's live + `wb.data.DataFrame` calls; a consumer selects its economies with `.loc`. # Constructed, tracking a moving source: the World Bank republishes and revises # this series continuously, and the file is meant to be refreshed in place on @@ -43,12 +44,13 @@ source: growth, annual %), from World Bank national accounts data and OECD National Accounts data files. Retrieved via wbgapi. note: > - Five economies and the wide `YR` layout are the lecture's choices, - not the source's — the builder passes exactly the lecture's arguments - (`wb.data.DataFrame('NY.GDP.MKTP.KD.ZG', ['USA','ARG','GBR','GRC','JPN'], - labels=True)`). Row order is whatever wbgapi returns (JPN, GRC, GBR, ARG, - USA on every run so far) and is kept as returned so a refresh diff is - values-only. + The economy set and the wide `YR` layout are the lecture's choices, + not the source's — the builder passes the union of the lecture's two + selections to `wb.data.DataFrame('NY.GDP.MKTP.KD.ZG', [...], labels=True)`. + Grew from five economies to nine on 2026-09-01, while the file had no + consumer, so that the lecture's second selection is covered too. Row + order is whatever wbgapi returns and is kept as returned so a refresh + diff is values-only. license: name: CC BY-4.0 @@ -83,7 +85,7 @@ maintainer: QuantEcon # to byte-match and no figure a change here could alter today. integrity: - sha256: 05b52b8d66be7565a75b66f897bf21095a7844fb0d47c036cad019bfc514315f + sha256: 41df23233ea238e4166c5c21cc383791015d4c9af6868b840901a8ce083c9da8 upstream: # `diverged`, not `failing`: the builder was re-run in full against live # WDI and its output differs from these bytes in the way a revised @@ -96,7 +98,7 @@ integrity: status: verified date: 2026-09-01 against: builders/business_cycle.py - note: "Refreshed 2026-09-01 by the scheduled refresh: these bytes are the builder's validated output from the live source that day, so they are verified by construction. Overlap window YR1960..YR2023: 236 of 320 cells revised, max |change| 1.5002." + note: "Refreshed 2026-09-01 by the scheduled refresh: these bytes are the builder's validated output from the live source that day, so they are verified by construction. Overlap window YR1960..YR2025: 0 of 325 cells revised, max |change| 0.0." # --------------------------------------------------------------------------- # Shape @@ -114,8 +116,8 @@ schema: - {name: economy, dtype: str, description: "ISO3 code — wbgapi's index, written as the CSV's first column; the lecture reads it with index_col=0"} - {name: Country, dtype: str, description: "country name, the label column wbgapi adds with labels=True"} - {pattern: 'YR\d{4}', dtype: float64, description: "annual real GDP growth, percent, one column per year on an unbroken grid from YR1960 to the newest year the source carries (`schema.date_range.end`). The builder's validate() bounds every value at ±50 and requires at least one |value| ≥ 1, so a units switch cannot ship; the observed range is deliberately not written here, since prose in a dynamic snapshot's manifest must not embed facts a refresh can change"} - row_count_floor: 5 # exact by construction: the builder asks for - # five economies and validate() asserts the + row_count_floor: 9 # exact by construction: the builder asks for + # nine economies and validate() asserts the # set. Columns, not rows, are what grow. # `end` is the newest year column in the COMMITTED bytes, stamped by # scripts/snapshots.py on every refresh — which is what makes a refresh PR's @@ -123,20 +125,20 @@ schema: date_range: {start: 1960, end: 2025} # Growth is undefined in the series' first year, so YR1960 is empty for all - # five economies and nothing else is. validate() asserts exactly this hole. - known_nulls: {YR1960: 5} + # nine economies. validate() additionally allows a null only before an + # economy's first observation or in the newest two years, never inside a + # series — none of either exists in this vintage. + known_nulls: {YR1960: 9} # --------------------------------------------------------------------------- # Consumers — how a correction knows what to rebuild # --------------------------------------------------------------------------- -# None. No lecture reads these bytes: intro's business_cycle.md (lines 83, 109 -# and 512 on 2026-09-01) and its lecture-wasm mirror make the live wbgapi call -# the builder reproduces, and the org-wide code search for this basename on -# 2026-09-01 returned only this repo. Whether those lectures adopt the snapshot -# — and whether the live call stays as the lesson, with the snapshot as the -# wasm twin (AGENTS.md, "Live APIs") — is the P4 decision (PLAN Phase 8), not -# something this manifest settles. When a consumer is added, `cadence` below -# becomes load-bearing. +# None yet. Decided 2026-09-01: lecture-python-intro KEEPS its live wbgapi +# calls (the API is the lesson there), and lecture-wasm — where business_cycle +# is currently excluded from the build because pyodide cannot reach the API — +# will read this file and its two siblings plus us_business_cycle_monthly.csv +# (QuantEcon/lecture-wasm#70). When that repoint lands, record it here with +# `on_refresh: rebuild`, and `cadence` below becomes load-bearing. consumers: [] builder: builders/business_cycle.py @@ -148,4 +150,6 @@ builder_status: committed # four-stage since 2026-09-01 (PLAN Phase 5); # Dynamic snapshots only. The series is annual; the World Bank revises it # several times a year but the lecture's figures change materially only when # a new year lands (WDI typically carries year T from mid T+1). -cadence: annual +cadence: annual # shared with unemployment_rate_annual.csv and + # private_credit_to_gdp.csv, which the same + # builder writes in the same run diff --git a/lectures/private_credit_to_gdp.csv b/lectures/private_credit_to_gdp.csv new file mode 100644 index 0000000..f2662ae --- /dev/null +++ b/lectures/private_credit_to_gdp.csv @@ -0,0 +1,2 @@ +economy,Country,YR1960,YR1961,YR1962,YR1963,YR1964,YR1965,YR1966,YR1967,YR1968,YR1969,YR1970,YR1971,YR1972,YR1973,YR1974,YR1975,YR1976,YR1977,YR1978,YR1979,YR1980,YR1981,YR1982,YR1983,YR1984,YR1985,YR1986,YR1987,YR1988,YR1989,YR1990,YR1991,YR1992,YR1993,YR1994,YR1995,YR1996,YR1997,YR1998,YR1999,YR2000,YR2001,YR2002,YR2003,YR2004,YR2005,YR2006,YR2007,YR2008,YR2009,YR2010,YR2011,YR2012,YR2013,YR2014,YR2015,YR2016,YR2017,YR2018,YR2019,YR2020,YR2021,YR2022,YR2023,YR2024,YR2025 +GBR,United Kingdom,17.3389409290767,17.493246893571,17.4380535548127,19.343360828077,20.6720645371771,20.623659462135,19.4965958324737,19.82421875,19.7104677060134,19.350052548066,19.8159813410222,20.7525468287874,28.5508566726423,33.5649031732702,35.1995282537451,28.3141460914765,27.3619157976053,26.0847655421175,25.5990134902919,25.86194420458,26.2158726763391,30.8601371313897,34.0485865664453,37.0723310454147,42.2356793276187,43.9062454918151,77.6106416839909,82.7710460232693,91.5754459408487,102.930035021323,104.805960306851,102.333764425911,101.677998125474,99.876224683285,100.030864363201,96.9838148788017,99.8119545669975,101.512510184771,102.081363855748,105.111743558731,114.294248790913,119.336886441347,124.533460121241,129.216112132892,136.667035880836,142.664487913407,153.471395323663,169.583348638841,190.146653043876,191.129828943179,183.762819228277,169.61587364494,160.010705514872,148.906248602627,134.605055432715,130.08273790259,131.480106047871,132.333248459152,132.783359788981,131.443065370417,145.572075379344,136.161594591011,126.126397693252,117.978361754605,112.372018963535, diff --git a/lectures/private_credit_to_gdp.csv.yml b/lectures/private_credit_to_gdp.csv.yml new file mode 100644 index 0000000..6125ab5 --- /dev/null +++ b/lectures/private_credit_to_gdp.csv.yml @@ -0,0 +1,79 @@ +# Manifest for private_credit_to_gdp.csv — the third of three World Bank +# tables builders/business_cycle.py writes for the intro `business_cycle` +# lecture. The FILENAME IS PROVISIONAL (naming policy: QuantEcon/data-lectures#113) +# and free to change while `consumers` is empty. Landed 2026-09-01 as part of +# the P4 dynamic-snapshot pilot. + +filename: private_credit_to_gdp.csv +title: World Bank domestic credit to the private sector (% of GDP) — United Kingdom, 1960 onward +description: > + Annual domestic credit to the private sector as a percentage of GDP for the + United Kingdom — the one economy the intro `business_cycle` lecture uses to + illustrate credit over the cycle — from World Bank WDI series + FS.AST.PRVT.GD.ZS, in the wide layout wbgapi emits: one row (ISO3 code as + the index, country name as `Country`), one `YR` column per year from + 1960 to the newest year the source carries (`schema.date_range.end`). The + snapshot twin of the lecture's live call. + +class: dynamic-snapshot + +source: + name: World Bank, World Development Indicators (International Monetary Fund, International Financial Statistics, and World Bank and OECD GDP estimates) + series: FS.AST.PRVT.GD.ZS + url: https://data.worldbank.org/indicator/FS.AST.PRVT.GD.ZS + doi: null + version: > + Live WDI at each refresh; the vintage is pinned by `retrieved` and + integrity.upstream. + citation: > + World Bank, World Development Indicators, series FS.AST.PRVT.GD.ZS + (domestic credit to private sector, % of GDP), from the IMF's + International Financial Statistics and World Bank and OECD GDP + estimates. Retrieved via wbgapi. + note: > + The newest year is typically unpublished for this series at the time of + a refresh (YR2025 was empty on 2026-09-01); the builder allows a null in + the newest two years and nowhere inside the series. This ratio is + rebased in larger steps than GDP growth is revised, so the builder's + overlap bound is 25 points rather than 5. + +license: + name: CC BY-4.0 + url: https://datacatalog.worldbank.org/public-licenses#cc-by + redistribution: permitted + verified: 2026-09-01 + note: > + World Bank open data under CC BY-4.0, as for the two sibling files. + +# Stamped by scripts/snapshots.py on every merged refresh. +retrieved: 2026-09-01 +maintainer: QuantEcon + +integrity: + sha256: 197eaca1d173485aaad99fa0d45446fc69b11888c605c2592aeddce58dedc90d + upstream: + # Stamped by scripts/snapshots.py on every merged refresh. + status: verified + date: 2026-09-01 + against: builders/business_cycle.py + note: "Refreshed 2026-09-01 by the scheduled refresh: these bytes are the builder's validated output from the live source that day, so they are verified by construction. Overlap window n/a: 0 of 0 cells revised, max |change| 0." + +schema: + format: csv + columns: + - {name: economy, dtype: str, description: "ISO3 code — wbgapi's index, written as the CSV's first column; read it with index_col=0"} + - {name: Country, dtype: str, description: "country name, the label column wbgapi adds with labels=True"} + - {pattern: 'YR\d{4}', dtype: float64, description: "domestic credit to the private sector, % of GDP, one column per year on an unbroken grid from YR1960 to the newest year the source carries (`schema.date_range.end`). The builder's validate() bounds every value in [0, 400]"} + row_count_floor: 1 # exact by construction: one economy + # `end` is stamped by scripts/snapshots.py on every refresh. + date_range: {start: 1960, end: 2025} + # The newest year may be unpublished (a trailing null); nothing else. + # validate() asserts placement, not a count, which is why this is a ceiling. + known_nulls_total: 2 + +# None yet — see business_cycle_data.csv.yml (QuantEcon/lecture-wasm#70). +consumers: [] + +builder: builders/business_cycle.py +builder_status: committed +cadence: annual diff --git a/lectures/unemployment_rate_annual.csv b/lectures/unemployment_rate_annual.csv new file mode 100644 index 0000000..3a1a1f9 --- /dev/null +++ b/lectures/unemployment_rate_annual.csv @@ -0,0 +1,5 @@ +economy,Country,YR1960,YR1961,YR1962,YR1963,YR1964,YR1965,YR1966,YR1967,YR1968,YR1969,YR1970,YR1971,YR1972,YR1973,YR1974,YR1975,YR1976,YR1977,YR1978,YR1979,YR1980,YR1981,YR1982,YR1983,YR1984,YR1985,YR1986,YR1987,YR1988,YR1989,YR1990,YR1991,YR1992,YR1993,YR1994,YR1995,YR1996,YR1997,YR1998,YR1999,YR2000,YR2001,YR2002,YR2003,YR2004,YR2005,YR2006,YR2007,YR2008,YR2009,YR2010,YR2011,YR2012,YR2013,YR2014,YR2015,YR2016,YR2017,YR2018,YR2019,YR2020,YR2021,YR2022,YR2023,YR2024,YR2025 +JPN,Japan,1.7,1.4,1.3,1.3,1.1,1.2,1.3,1.3,1.2,1.1,1.1,1.2,1.4,1.3,1.4,1.9,2.0,2.0,2.2,2.1,2.0,2.2,2.4,2.6,2.7,2.6,2.8,2.8,2.5,2.3,2.1,2.1,2.2,2.5,2.9,3.2,3.4,3.4,4.1,4.7,4.748,5.02,5.386,5.251,4.734,4.446,4.192,3.888,4.002,5.068,5.102,4.55,4.358,4.038,3.589,3.385,3.132,2.822,2.467,2.351,2.809,2.808,2.614,2.586,2.5,2.5 +GBR,United Kingdom,,,,,,,,,,,,3.3,3.7,2.6,2.6,4.0,5.5,5.8,4.991,3.701,3.882,10.4,10.9,11.088,10.901,11.48,11.508,11.017,9.004,7.405,6.969,8.55,9.772,10.346,9.648,8.694,8.189,7.072,6.198,6.042,5.558,4.696,5.037,4.807,4.594,4.884,5.472,5.399,5.748,7.683,7.973,8.195,8.279,7.752,6.398,5.552,4.909,4.501,4.145,3.657,4.518,4.864,3.768,4.025,4.361,4.896 +FRA,France,,,,,,,,,,,2.418,2.671,2.774,2.686,2.862,4.082,4.466,5.013,5.274,6.03,6.422,7.542,8.201,7.918,9.533,10.259,10.23,10.735,10.18,9.618,9.36,9.134,10.203,11.32,12.593,11.834,12.367,12.566,12.074,11.981,10.218,8.61,8.702,8.306,8.914,8.882,8.832,8.008,7.386,9.122,9.279,9.228,9.841,9.913,10.273,10.354,10.057,9.409,9.018,8.415,8.009,7.871,7.303,7.335,7.436,7.7 +USA,United States,5.5,6.7,5.5,5.7,5.2,4.5,3.8,3.8,3.6,3.5,4.9,5.9,5.6,4.9,5.6,8.5,7.7,7.1,6.1,5.8,7.1,7.6,9.7,9.6,7.5,7.2,7.0,6.2,5.5,5.3,5.6,6.8,7.5,6.9,6.119,5.65,5.451,5.0,4.511,4.219,3.992,4.731,5.783,5.989,5.529,5.084,4.623,4.622,5.784,9.254,9.633,8.949,8.069,7.375,6.168,5.28,4.869,4.355,3.896,3.669,8.055,5.349,3.65,3.638,4.022,4.282 diff --git a/lectures/unemployment_rate_annual.csv.yml b/lectures/unemployment_rate_annual.csv.yml new file mode 100644 index 0000000..e31ea97 --- /dev/null +++ b/lectures/unemployment_rate_annual.csv.yml @@ -0,0 +1,90 @@ +# Manifest for unemployment_rate_annual.csv — the second of three World Bank +# tables builders/business_cycle.py writes for the intro `business_cycle` +# lecture (with business_cycle_data.csv and private_credit_to_gdp.csv). The +# FILENAME IS PROVISIONAL: it follows the strawman on the naming-policy issue +# (QuantEcon/data-lectures#113) and is free to change while `consumers` is +# empty. Landed 2026-09-01 as part of the P4 dynamic-snapshot pilot. + +filename: unemployment_rate_annual.csv +title: World Bank unemployment rate (annual, % of labour force) — USA, FRA, GBR, JPN, 1960 onward +description: > + Annual unemployment as a percentage of the total labour force (national + estimate) for the United States, France, the United Kingdom and Japan — + the four economies the intro `business_cycle` lecture compares — from World + Bank WDI series SL.UEM.TOTL.NE.ZS, in the wide layout wbgapi emits: one row + per economy (ISO3 code as the index, country name as `Country`), one + `YR` column per year from 1960 to the newest year the source carries + (`schema.date_range.end`). The snapshot twin of the lecture's live call. + +class: dynamic-snapshot + +source: + name: World Bank, World Development Indicators (International Labour Organization, ILOSTAT database, national estimates) + series: SL.UEM.TOTL.NE.ZS + url: https://data.worldbank.org/indicator/SL.UEM.TOTL.NE.ZS + doi: null + version: > + Live WDI at each refresh; the vintage is pinned by `retrieved` and + integrity.upstream, not by a source-side edition (WDI carries only a + database-level lastupdated stamp). + citation: > + World Bank, World Development Indicators, series SL.UEM.TOTL.NE.ZS + (unemployment, total, % of total labor force, national estimate), from + the International Labour Organization's ILOSTAT database. Retrieved via + wbgapi. + note: > + National estimates, so the series start when each country's own labour + force survey does: the United Kingdom from 1971 and France from 1970, + the United States and Japan from 1960. The builder allows a null only + before an economy's first observation or in the newest two years, so + those leading gaps are structural and a gap opening later is a failure. + +license: + name: CC BY-4.0 + url: https://datacatalog.worldbank.org/public-licenses#cc-by + redistribution: permitted + verified: 2026-09-01 + note: > + World Bank open data under CC BY-4.0, the same terms as + business_cycle_data.csv (stated in the series metadata the builder dumps + to provenance/ for the GDP series; WDI applies one licence across the + database). + +# Stamped by scripts/snapshots.py on every merged refresh. +retrieved: 2026-09-01 +maintainer: QuantEcon + +integrity: + sha256: 74a83c3cec97aea3899cc9f4c8e357b2527af4342d03ac3f3a8ec31b1e70fc71 + upstream: + # Stamped by scripts/snapshots.py on every merged refresh: a fresh + # snapshot is `verified` by construction — these bytes are the builder's + # validated output from the live source that day. + status: verified + date: 2026-09-01 + against: builders/business_cycle.py + note: "Refreshed 2026-09-01 by the scheduled refresh: these bytes are the builder's validated output from the live source that day, so they are verified by construction. Overlap window n/a: 0 of 0 cells revised, max |change| 0." + +schema: + format: csv + columns: + - {name: economy, dtype: str, description: "ISO3 code — wbgapi's index, written as the CSV's first column; read it with index_col=0"} + - {name: Country, dtype: str, description: "country name, the label column wbgapi adds with labels=True"} + - {pattern: 'YR\d{4}', dtype: float64, description: "unemployment, % of the labour force, one column per year on an unbroken grid from YR1960 to the newest year the source carries (`schema.date_range.end`). The builder's validate() bounds every value in [0, 60]; the observed range is deliberately not written here"} + row_count_floor: 4 # exact by construction: four economies + # `end` is stamped by scripts/snapshots.py on every refresh. + date_range: {start: 1960, end: 2025} + # Leading nulls only: GBR is empty 1960–1970 (11 cells) and FRA 1960–1969 + # (10), where the national series had not started. Declared as a total + # because the holes are per economy, not per column; validate() asserts + # their placement (before the first observation), not this count. + known_nulls_total: 21 + +# None yet — see business_cycle_data.csv.yml: lecture-wasm will read this +# file once business_cycle is re-enabled there (QuantEcon/lecture-wasm#70); +# lecture-python-intro keeps its live call. +consumers: [] + +builder: builders/business_cycle.py +builder_status: committed +cadence: annual diff --git a/lectures/us_business_cycle_monthly.csv b/lectures/us_business_cycle_monthly.csv new file mode 100644 index 0000000..fc20d3e --- /dev/null +++ b/lectures/us_business_cycle_monthly.csv @@ -0,0 +1,1292 @@ +DATE,UNRATE,USREC,UMCSENT,CPILFESL,INDPRO,M0892AUSM156SNBR +1919-01-01,,1,,,4.8739, +1919-02-01,,1,,,4.6585, +1919-03-01,,1,,,4.5238, +1919-04-01,,0,,,4.6046, +1919-05-01,,0,,,4.6315, +1919-06-01,,0,,,4.9277, +1919-07-01,,0,,,5.2239, +1919-08-01,,0,,,5.3047, +1919-09-01,,0,,,5.197, +1919-10-01,,0,,,5.1432, +1919-11-01,,0,,,5.0624, +1919-12-01,,0,,,5.1432, +1920-01-01,,0,,,5.6279, +1920-02-01,,1,,,5.6279, +1920-03-01,,1,,,5.5201, +1920-04-01,,1,,,5.2239, +1920-05-01,,1,,,5.3586, +1920-06-01,,1,,,5.4124, +1920-07-01,,1,,,5.2778, +1920-08-01,,1,,,5.3047, +1920-09-01,,1,,,5.1162, +1920-10-01,,1,,,4.9008, +1920-11-01,,1,,,4.4969, +1920-12-01,,1,,,4.2276, +1921-01-01,,1,,,3.9853, +1921-02-01,,1,,,3.9045, +1921-03-01,,1,,,3.7968, +1921-04-01,,1,,,3.7968, +1921-05-01,,1,,,3.9045, +1921-06-01,,1,,,3.8776, +1921-07-01,,1,,,3.8506, +1921-08-01,,0,,,3.9853, +1921-09-01,,0,,,4.0122, +1921-10-01,,0,,,4.2545, +1921-11-01,,0,,,4.2007, +1921-12-01,,0,,,4.1738, +1922-01-01,,0,,,4.3353, +1922-02-01,,0,,,4.5238, +1922-03-01,,0,,,4.7662, +1922-04-01,,0,,,4.6046, +1922-05-01,,0,,,4.847, +1922-06-01,,0,,,5.0893, +1922-07-01,,0,,,5.0893, +1922-08-01,,0,,,4.9816, +1922-09-01,,0,,,5.2509, +1922-10-01,,0,,,5.5471, +1922-11-01,,0,,,5.7894, +1922-12-01,,0,,,5.951, +1923-01-01,,0,,,5.8163, +1923-02-01,,0,,,5.8971, +1923-03-01,,0,,,6.0856, +1923-04-01,,0,,,6.2203, +1923-05-01,,0,,,6.301, +1923-06-01,,1,,,6.2472, +1923-07-01,,1,,,6.1933, +1923-08-01,,1,,,6.0856, +1923-09-01,,1,,,5.951, +1923-10-01,,1,,,5.9241, +1923-11-01,,1,,,5.9241, +1923-12-01,,1,,,5.7894, +1924-01-01,,1,,,5.924, +1924-02-01,,1,,,6.0318, +1924-03-01,,1,,,5.9241, +1924-04-01,,1,,,5.7356, +1924-05-01,,1,,,5.4932, +1924-06-01,,1,,,5.2509, +1924-07-01,,1,,,5.1701, +1924-08-01,,0,,,5.3586, +1924-09-01,,0,,,5.5471, +1924-10-01,,0,,,5.6817, +1924-11-01,,0,,,5.7894, +1924-12-01,,0,,,5.951, +1925-01-01,,0,,,6.1395, +1925-02-01,,0,,,6.1395, +1925-03-01,,0,,,6.1395, +1925-04-01,,0,,,6.1933, +1925-05-01,,0,,,6.1664, +1925-06-01,,0,,,6.1125, +1925-07-01,,0,,,6.2741, +1925-08-01,,0,,,6.1664, +1925-09-01,,0,,,6.0856, +1925-10-01,,0,,,6.328, +1925-11-01,,0,,,6.4626, +1925-12-01,,0,,,6.5434, +1926-01-01,,0,,,6.4357, +1926-02-01,,0,,,6.4357, +1926-03-01,,0,,,6.5165, +1926-04-01,,0,,,6.5165, +1926-05-01,,0,,,6.4626, +1926-06-01,,0,,,6.5434, +1926-07-01,,0,,,6.5703, +1926-08-01,,0,,,6.6511, +1926-09-01,,0,,,6.7588, +1926-10-01,,0,,,6.7588, +1926-11-01,,1,,,6.7319, +1926-12-01,,1,,,6.705, +1927-01-01,,1,,,6.678, +1927-02-01,,1,,,6.7319, +1927-03-01,,1,,,6.8127, +1927-04-01,,1,,,6.6511, +1927-05-01,,1,,,6.705, +1927-06-01,,1,,,6.678, +1927-07-01,,1,,,6.5972, +1927-08-01,,1,,,6.5972, +1927-09-01,,1,,,6.4895, +1927-10-01,,1,,,6.3549, +1927-11-01,,1,,,6.3549, +1927-12-01,,0,,,6.3818, +1928-01-01,,0,,,6.5165, +1928-02-01,,0,,,6.5703, +1928-03-01,,0,,,6.6242, +1928-04-01,,0,,,6.5972, +1928-05-01,,0,,,6.678, +1928-06-01,,0,,,6.7319, +1928-07-01,,0,,,6.8127, +1928-08-01,,0,,,6.9473, +1928-09-01,,0,,,7.0012, +1928-10-01,,0,,,7.1358, +1928-11-01,,0,,,7.2704, +1928-12-01,,0,,,7.4051, +1929-01-01,,0,,,7.5128, +1929-02-01,,0,,,7.4858, +1929-03-01,,0,,,7.5128, +1929-04-01,,0,,,7.6474,0.69 +1929-05-01,,0,,,7.7821,1.65 +1929-06-01,,0,,,7.8359,2.06 +1929-07-01,,0,,,7.9436,0.79 +1929-08-01,,0,,,7.8628,0.04 +1929-09-01,,1,,,7.809,0.91 +1929-10-01,,1,,,7.6743,2.31 +1929-11-01,,1,,,7.2974,1.89 +1929-12-01,,1,,,6.9742,2.08 +1930-01-01,,1,,,6.9742,2.21 +1930-02-01,,1,,,6.9473,3.12 +1930-03-01,,1,,,6.8396,3.63 +1930-04-01,,1,,,6.7857,3.6 +1930-05-01,,1,,,6.678,3.39 +1930-06-01,,1,,,6.4895,3.77 +1930-07-01,,1,,,6.1933,5.02 +1930-08-01,,1,,,6.0587,6.64 +1930-09-01,,1,,,5.951,7.79 +1930-10-01,,1,,,5.7894,9.01 +1930-11-01,,1,,,5.6548,10.78 +1930-12-01,,1,,,5.5201,11.89 +1931-01-01,,1,,,5.4932,11.18 +1931-02-01,,1,,,5.5201,11.67 +1931-03-01,,1,,,5.6279,11.78 +1931-04-01,,1,,,5.6548,11.94 +1931-05-01,,1,,,5.574,12.3 +1931-06-01,,1,,,5.4394,13.02 +1931-07-01,,1,,,5.3586,13.84 +1931-08-01,,1,,,5.1701,15.01 +1931-09-01,,1,,,4.9277,15.83 +1931-10-01,,1,,,4.7392,16.75 +1931-11-01,,1,,,4.6854,17.95 +1931-12-01,,1,,,4.6585,19.1 +1932-01-01,,1,,,4.5238,18.04 +1932-02-01,,1,,,4.4161,18.66 +1932-03-01,,1,,,4.3623,19.78 +1932-04-01,,1,,,4.0661,21.03 +1932-05-01,,1,,,3.9314,22.34 +1932-06-01,,1,,,3.7968,23.72 +1932-07-01,,1,,,3.6891,24.54 +1932-08-01,,1,,,3.7968,25.02 +1932-09-01,,1,,,4.0391,24.85 +1932-10-01,,1,,,4.1738,24.82 +1932-11-01,,1,,,4.1738,24.67 +1932-12-01,,1,,,4.093,25.2 +1933-01-01,,1,,,4.0122,23.72 +1933-02-01,,1,,,4.0391,24.03 +1933-03-01,,1,,,3.7968,25.36 +1933-04-01,,0,,,4.0661,25.49 +1933-05-01,,0,,,4.7392,25.59 +1933-06-01,,0,,,5.4663,25.04 +1933-07-01,,0,,,5.9779,23.84 +1933-08-01,,0,,,5.7356,22.45 +1933-09-01,,0,,,5.4124,21.48 +1933-10-01,,0,,,5.1432,21.26 +1933-11-01,,0,,,4.847,21.29 +1933-12-01,,0,,,4.8739,21.07 +1934-01-01,,0,,,5.0354,18.8 +1934-02-01,,0,,,5.2778,17.9 +1934-03-01,,0,,,5.5201,16.81 +1934-04-01,,0,,,5.5201,16.64 +1934-05-01,,0,,,5.6278,16.96 +1934-06-01,,0,,,5.5201,17.69 +1934-07-01,,0,,,5.1432,18.79 +1934-08-01,,0,,,5.0893,20.27 +1934-09-01,,0,,,4.7931,21.51 +1934-10-01,,0,,,5.0085,21.54 +1934-11-01,,0,,,5.0624,21.41 +1934-12-01,,0,,,5.3855,20.44 +1935-01-01,,0,,,5.8163,17.88 +1935-02-01,,0,,,5.9241,17.5 +1935-03-01,,0,,,5.8971,17.54 +1935-04-01,,0,,,5.7894,17.55 +1935-05-01,,0,,,5.7894,17.77 +1935-06-01,,0,,,5.8702,18.14 +1935-07-01,,0,,,5.8702,17.72 +1935-08-01,,0,,,6.0856,17.77 +1935-09-01,,0,,,6.2472,18.41 +1935-10-01,,0,,,6.4357,17.45 +1935-11-01,,0,,,6.5703,16.8 +1935-12-01,,0,,,6.6511,16.44 +1936-01-01,,0,,,6.5434,14.96 +1936-02-01,,0,,,6.3818,15.1 +1936-03-01,,0,,,6.4626,14.86 +1936-04-01,,0,,,6.8665,15.57 +1936-05-01,,0,,,7.0012,14.67 +1936-06-01,,0,,,7.1358,14.85 +1936-07-01,,0,,,7.2704,14.25 +1936-08-01,,0,,,7.3781,13.46 +1936-09-01,,0,,,7.5128,13.31 +1936-10-01,,0,,,7.6205,13.81 +1936-11-01,,0,,,7.8359,13.24 +1936-12-01,,0,,,8.0783,12.78 +1937-01-01,,0,,,8.0513,12.11 +1937-02-01,,0,,,8.159,11.99 +1937-03-01,,0,,,8.3475,11.73 +1937-04-01,,0,,,8.3475,11.85 +1937-05-01,,0,,,8.3745,11.23 +1937-06-01,,1,,,8.2667,11.5 +1937-07-01,,1,,,8.3206,10.97 +1937-08-01,,1,,,8.2667,11.14 +1937-09-01,,1,,,7.9975,11.63 +1937-10-01,,1,,,7.4051,12.19 +1937-11-01,,1,,,6.678,13.7 +1937-12-01,,1,,,6.0856,15.91 +1938-01-01,,1,,,5.951,16.88 +1938-02-01,,1,,,5.8971,17.88 +1938-03-01,,1,,,5.8971,18.49 +1938-04-01,,1,,,5.7894,18.96 +1938-05-01,,1,,,5.6548,19.74 +1938-06-01,,1,,,5.7086,20.0 +1938-07-01,,0,,,6.0318,19.58 +1938-08-01,,0,,,6.3549,19.18 +1938-09-01,,0,,,6.5434,18.6 +1938-10-01,,0,,,6.705,18.22 +1938-11-01,,0,,,6.9742,17.14 +1938-12-01,,0,,,7.055,16.36 +1939-01-01,,0,,,7.055,17.61 +1939-02-01,,0,,,7.1089,17.43 +1939-03-01,,0,,,7.1358,17.19 +1939-04-01,,0,,,7.1089,17.39 +1939-05-01,,0,,,7.0819,16.9 +1939-06-01,,0,,,7.2435,16.29 +1939-07-01,,0,,,7.4589,16.18 +1939-08-01,,0,,,7.5666,15.79 +1939-09-01,,0,,,8.0244,15.22 +1939-10-01,,0,,,8.4283,15.48 +1939-11-01,,0,,,8.6437,15.49 +1939-12-01,,0,,,8.6437,15.19 +1940-01-01,,0,,,8.536,15.53 +1940-02-01,,0,,,8.2667,15.67 +1940-03-01,,0,,,8.0783,15.66 +1940-04-01,,0,,,8.2398,16.0 +1940-05-01,,0,,,8.4822,15.57 +1940-06-01,,0,,,8.7514,14.96 +1940-07-01,,0,,,8.8592,14.24 +1940-08-01,,0,,,8.913,13.35 +1940-09-01,,0,,,9.1015,12.55 +1940-10-01,,0,,,9.2361,12.26 +1940-11-01,,0,,,9.4516,11.65 +1940-12-01,,0,,,9.7747,10.85 +1941-01-01,,0,,,10.017,10.49 +1941-02-01,,0,,,10.3132,9.64 +1941-03-01,,0,,,10.6364,9.1 +1941-04-01,,0,,,10.6633,7.98 +1941-05-01,,0,,,11.148,6.84 +1941-06-01,,0,,,11.2288,5.66 +1941-07-01,,0,,,11.3634,3.88 +1941-08-01,,0,,,11.4981,3.36 +1941-09-01,,0,,,11.4981,3.43 +1941-10-01,,0,,,11.6058,4.13 +1941-11-01,,0,,,11.6596,3.93 +1941-12-01,,0,,,11.8481,3.58 +1942-01-01,,0,,,12.0905,3.72 +1942-02-01,,0,,,12.3059,3.56 +1942-03-01,,0,,,12.4405,3.22 +1942-04-01,,0,,,12.0905,2.33 +1942-05-01,,0,,,12.1174,1.22 +1942-06-01,,0,,,12.1443,0.24 +1942-07-01,,0,,,12.4405, +1942-08-01,,0,,,12.8175, +1942-09-01,,0,,,13.1137, +1942-10-01,,0,,,13.5445, +1942-11-01,,0,,,13.8677, +1942-12-01,,0,,,14.1908, +1943-01-01,,0,,,14.3255, +1943-02-01,,0,,,14.7024, +1943-03-01,,0,,,14.8102, +1943-04-01,,0,,,14.9986, +1943-05-01,,0,,,15.1063, +1943-06-01,,0,,,15.0256, +1943-07-01,,0,,,15.4833, +1943-08-01,,0,,,15.8064, +1943-09-01,,0,,,16.2104, +1943-10-01,,0,,,16.4527, +1943-11-01,,0,,,16.6681, +1943-12-01,,0,,,16.4527, +1944-01-01,,0,,,16.6412, +1944-02-01,,0,,,16.7758, +1944-03-01,,0,,,16.7489, +1944-04-01,,0,,,16.7489, +1944-05-01,,0,,,16.6412, +1944-06-01,,0,,,16.5874, +1944-07-01,,0,,,16.5604, +1944-08-01,,0,,,16.7758, +1944-09-01,,0,,,16.6681, +1944-10-01,,0,,,16.722, +1944-11-01,,0,,,16.5873, +1944-12-01,,0,,,16.5335, +1945-01-01,,0,,,16.3719, +1945-02-01,,0,,,16.3181, +1945-03-01,,1,,,16.2104, +1945-04-01,,1,,,15.9142, +1945-05-01,,1,,,15.4833, +1945-06-01,,1,,,15.1333, +1945-07-01,,1,,,14.7832, +1945-08-01,,1,,,13.2483, +1945-09-01,,1,,,12.0635, +1945-10-01,,1,,,11.5788, +1945-11-01,,0,,,12.0097, +1945-12-01,,0,,,12.0635, +1946-01-01,,0,,,11.3903, +1946-02-01,,0,,,10.8249, +1946-03-01,,0,,,11.9558, +1946-04-01,,0,,,11.7404, +1946-05-01,,0,,,11.3096, +1946-06-01,,0,,,12.0097, +1946-07-01,,0,,,12.4136, +1946-08-01,,0,,,12.8714, +1946-09-01,,0,,,13.1137, +1946-10-01,,0,,,13.356, +1946-11-01,,0,,,13.4368, +1946-12-01,,0,,,13.5176, +1947-01-01,,0,,,13.6792, +1947-02-01,,0,,,13.76, +1947-03-01,,0,,,13.8407, +1947-04-01,,0,,,13.733, +1947-05-01,,0,,,13.7869, +1947-06-01,,0,,,13.7869, +1947-07-01,,0,,,13.7061, +1947-08-01,,0,,,13.7869, +1947-09-01,,0,,,13.8946, +1947-10-01,,0,,,14.0292, +1947-11-01,,0,,,14.2177, +1947-12-01,,0,,,14.2716, +1948-01-01,3.4,0,,,14.3524, +1948-02-01,3.8,0,,,14.3793, +1948-03-01,4.0,0,,,14.2177, +1948-04-01,3.9,0,,,14.2447, +1948-05-01,3.5,0,,,14.487, +1948-06-01,3.6,0,,,14.6755, +1948-07-01,3.6,0,,,14.6755, +1948-08-01,3.9,0,,,14.6216, +1948-09-01,3.8,0,,,14.5139, +1948-10-01,3.7,0,,,14.6217, +1948-11-01,3.8,0,,,14.4332, +1948-12-01,4.0,1,,,14.2985, +1949-01-01,4.3,1,,,14.1639, +1949-02-01,4.7,1,,,14.0292, +1949-03-01,5.0,1,,,13.76, +1949-04-01,5.3,1,,,13.6792, +1949-05-01,6.1,1,,,13.4907, +1949-06-01,6.2,1,,,13.4638, +1949-07-01,6.7,1,,,13.4368, +1949-08-01,6.8,1,,,13.5715, +1949-09-01,6.6,1,,,13.7061, +1949-10-01,7.9,1,,,13.1945, +1949-11-01,6.4,0,,,13.5446, +1949-12-01,6.6,0,,,13.7869, +1950-01-01,6.5,0,,,14.0292, +1950-02-01,6.4,0,,,14.0831, +1950-03-01,6.3,0,,,14.5409, +1950-04-01,5.8,0,,,15.0255, +1950-05-01,5.5,0,,,15.3756, +1950-06-01,5.4,0,,,15.8334, +1950-07-01,5.0,0,,,16.345, +1950-08-01,4.5,0,,,16.8566, +1950-09-01,4.4,0,,,16.7489, +1950-10-01,4.2,0,,,16.8566, +1950-11-01,4.2,0,,,16.8297, +1950-12-01,4.3,0,,,17.1259, +1951-01-01,3.7,0,,,17.1797, +1951-02-01,3.4,0,,,17.2875, +1951-03-01,3.4,0,,,17.3682, +1951-04-01,3.1,0,,,17.3952, +1951-05-01,3.0,0,,,17.3413, +1951-06-01,3.2,0,,,17.2605, +1951-07-01,3.1,0,,,16.9913, +1951-08-01,3.1,0,,,16.8297, +1951-09-01,3.3,0,,,16.9374, +1951-10-01,3.5,0,,,16.9105, +1951-11-01,3.5,0,,,17.0451, +1951-12-01,3.1,0,,,17.1528, +1952-01-01,3.2,0,,,17.3413, +1952-02-01,3.1,0,,,17.449, +1952-03-01,2.9,0,,,17.5029, +1952-04-01,2.9,0,,,17.3413, +1952-05-01,3.0,0,,,17.1798, +1952-06-01,3.0,0,,,17.0182, +1952-07-01,3.2,0,,,16.7489, +1952-08-01,3.4,0,,,17.826, +1952-09-01,3.1,0,,,18.4723, +1952-10-01,3.0,0,,,18.6608, +1952-11-01,2.8,0,86.2,,19.0378, +1952-12-01,2.7,0,,,19.1455, +1953-01-01,2.9,0,,,19.1993, +1953-02-01,2.6,0,90.7,,19.307, +1953-03-01,2.6,0,,,19.4686, +1953-04-01,2.7,0,,,19.5494, +1953-05-01,2.5,0,,,19.6571, +1953-06-01,2.5,0,,,19.5763, +1953-07-01,2.6,0,,,19.8187, +1953-08-01,2.7,1,80.8,,19.7109, +1953-09-01,2.9,1,,,19.307, +1953-10-01,3.1,1,,,19.1455, +1953-11-01,3.5,1,80.7,,18.6877, +1953-12-01,4.5,1,,,18.2299, +1954-01-01,4.9,1,,,18.0953, +1954-02-01,5.2,1,82.0,,18.1492, +1954-03-01,5.7,1,,,18.0414, +1954-04-01,5.9,1,,,17.9337, +1954-05-01,5.9,1,82.9,,18.0414, +1954-06-01,5.6,0,,,18.0953, +1954-07-01,5.8,0,,,18.1222, +1954-08-01,6.0,0,,,18.0953, +1954-09-01,6.1,0,,,18.1222, +1954-10-01,5.7,0,,,18.3377, +1954-11-01,5.3,0,87.0,,18.6338, +1954-12-01,5.0,0,,,18.8762, +1955-01-01,4.9,0,,,19.307, +1955-02-01,4.7,0,95.9,,19.5494, +1955-03-01,4.6,0,,,20.0071, +1955-04-01,4.7,0,,,20.2495, +1955-05-01,4.3,0,99.1,,20.5726, +1955-06-01,4.2,0,,,20.5996, +1955-07-01,4.0,0,,,20.7611, +1955-08-01,4.2,0,,,20.7342, +1955-09-01,4.1,0,,,20.8688, +1955-10-01,4.3,0,,,21.2189, +1955-11-01,4.2,0,99.7,,21.2727, +1955-12-01,4.2,0,,,21.3535, +1956-01-01,4.0,0,,,21.4882, +1956-02-01,3.9,0,,,21.2997, +1956-03-01,4.2,0,,,21.2997, +1956-04-01,4.0,0,,,21.4612, +1956-05-01,4.3,0,98.2,,21.2728, +1956-06-01,4.3,0,,,21.0843, +1956-07-01,4.4,0,,,20.438, +1956-08-01,4.1,0,99.9,,21.2727, +1956-09-01,3.9,0,,,21.7574, +1956-10-01,3.9,0,,,21.9459, +1956-11-01,4.3,0,100.2,,21.7574, +1956-12-01,4.2,0,,,22.0806, +1957-01-01,4.2,0,,28.5,21.9998, +1957-02-01,3.9,0,,28.6,22.2152, +1957-03-01,3.7,0,,28.7,22.1883, +1957-04-01,3.9,0,,28.8,21.8921, +1957-05-01,4.1,0,92.9,28.8,21.8113, +1957-06-01,4.3,0,,28.9,21.8652, +1957-07-01,4.2,0,,29.0,21.9998, +1957-08-01,4.1,0,,29.0,21.9998, +1957-09-01,4.4,1,,29.1,21.8113, +1957-10-01,4.5,1,,29.2,21.4882, +1957-11-01,5.1,1,83.7,29.3,20.9765, +1957-12-01,5.2,1,,29.3,20.5726, +1958-01-01,5.8,1,,29.3,20.1956, +1958-02-01,6.4,1,78.5,29.4,19.7648, +1958-03-01,6.7,1,,29.5,19.5225, +1958-04-01,7.4,1,,29.5,19.1993, +1958-05-01,7.4,0,80.9,29.5,19.3878, +1958-06-01,7.3,0,,29.6,19.8994, +1958-07-01,7.5,0,,29.6,20.1956, +1958-08-01,7.4,0,,29.6,20.5995, +1958-09-01,7.1,0,,29.7,20.7881, +1958-10-01,6.7,0,,29.7,21.0304, +1958-11-01,6.2,0,90.8,29.8,21.6497, +1958-12-01,6.2,0,,29.9,21.6767, +1959-01-01,6.0,0,,29.9,21.9998, +1959-02-01,5.9,0,,29.9,22.4306, +1959-03-01,5.6,0,,30.0,22.7538, +1959-04-01,5.2,0,,30.0,23.2384, +1959-05-01,5.1,0,95.3,30.1,23.5885, +1959-06-01,5.0,0,,30.2,23.6155, +1959-07-01,5.1,0,,30.2,23.05, +1959-08-01,5.2,0,,30.2,22.2691, +1959-09-01,5.5,0,,30.3,22.2421, +1959-10-01,5.7,0,,30.4,22.0806, +1959-11-01,5.8,0,93.8,30.4,22.2152, +1959-12-01,5.3,0,,30.5,23.5885, +1960-01-01,5.2,0,,30.5,24.2078, +1960-02-01,4.8,0,100.0,30.6,23.9924, +1960-03-01,5.4,0,,30.6,23.777, +1960-04-01,5.2,0,,30.6,23.5885, +1960-05-01,5.1,1,93.3,30.6,23.5616, +1960-06-01,5.4,1,,30.7,23.2654, +1960-07-01,5.5,1,,30.6,23.1846, +1960-08-01,5.6,1,97.2,30.6,23.1577, +1960-09-01,5.5,1,,30.6,22.9153, +1960-10-01,6.1,1,,30.8,22.8884, +1960-11-01,6.1,1,90.1,30.8,22.5653, +1960-12-01,6.6,1,,30.7,22.1344, +1961-01-01,6.6,1,,30.8,22.1613, +1961-02-01,6.9,1,91.6,30.8,22.1344, +1961-03-01,6.9,0,,30.9,22.2691, +1961-04-01,7.0,0,,30.9,22.7268, +1961-05-01,7.1,0,92.5,30.9,23.0769, +1961-06-01,6.9,0,,31.0,23.4, +1961-07-01,7.0,0,,31.0,23.6693, +1961-08-01,6.6,0,99.2,31.1,23.8847, +1961-09-01,6.7,0,,31.1,23.8578, +1961-10-01,6.5,0,,31.1,24.3156, +1961-11-01,6.1,0,93.0,31.2,24.6926, +1961-12-01,6.0,0,,31.2,24.908, +1962-01-01,5.8,0,,31.2,24.6925, +1962-02-01,5.5,0,99.9,31.2,25.0964, +1962-03-01,5.6,0,,31.3,25.2311, +1962-04-01,5.6,0,,31.3,25.285, +1962-05-01,5.5,0,95.4,31.4,25.258, +1962-06-01,5.5,0,,31.4,25.2042, +1962-07-01,5.4,0,,31.4,25.4465, +1962-08-01,5.7,0,91.6,31.5,25.4734, +1962-09-01,5.6,0,,31.5,25.635, +1962-10-01,5.4,0,,31.5,25.6619, +1962-11-01,5.7,0,95.0,31.5,25.7696, +1962-12-01,5.5,0,,31.6,25.7696, +1963-01-01,5.7,0,,31.5,25.9581, +1963-02-01,5.9,0,98.4,31.6,26.2544, +1963-03-01,5.7,0,,31.7,26.4159, +1963-04-01,5.7,0,,31.7,26.6582, +1963-05-01,5.9,0,91.7,31.7,26.9814, +1963-06-01,5.6,0,,31.8,27.0622, +1963-07-01,5.6,0,,31.8,26.9545, +1963-08-01,5.4,0,96.4,31.9,27.0083, +1963-09-01,5.5,0,,31.9,27.2776, +1963-10-01,5.5,0,,32.0,27.4661, +1963-11-01,5.7,0,94.4,32.0,27.6007, +1963-12-01,5.5,0,,32.1,27.5469, +1964-01-01,5.6,0,,32.2,27.7892, +1964-02-01,5.4,0,99.5,32.2,27.9777, +1964-03-01,5.4,0,,32.2,27.9777, +1964-04-01,5.3,0,,32.2,28.4355, +1964-05-01,5.1,0,98.5,32.2,28.597, +1964-06-01,5.2,0,,32.3,28.6778, +1964-07-01,4.9,0,,32.3,28.8663, +1964-08-01,5.0,0,100.6,32.3,29.0548, +1964-09-01,5.1,0,,32.3,29.1625, +1964-10-01,5.1,0,,32.4,28.7586, +1964-11-01,4.8,0,99.9,32.5,29.6472, +1964-12-01,5.0,0,,32.5,29.9973, +1965-01-01,4.9,0,,32.6,30.3204, +1965-02-01,5.1,0,102.0,32.6,30.5089, +1965-03-01,4.7,0,,32.6,30.9128, +1965-04-01,4.8,0,,32.7,31.0474, +1965-05-01,4.6,0,105.4,32.7,31.2898, +1965-06-01,4.6,0,,32.7,31.5321, +1965-07-01,4.4,0,,32.7,31.8283, +1965-08-01,4.4,0,103.4,32.7,31.963, +1965-09-01,4.3,0,,32.8,32.0438, +1965-10-01,4.2,0,,32.8,32.3669, +1965-11-01,4.1,0,102.9,32.9,32.5015, +1965-12-01,4.0,0,,33.0,32.9054, +1966-01-01,4.0,0,,33.0,33.2286, +1966-02-01,3.8,0,100.0,33.1,33.444, +1966-03-01,3.8,0,,33.1,33.9018, +1966-04-01,3.8,0,,33.3,33.9556, +1966-05-01,3.9,0,95.7,33.4,34.2787, +1966-06-01,3.8,0,,33.5,34.4403, +1966-07-01,3.8,0,,33.6,34.6288, +1966-08-01,3.8,0,91.2,33.7,34.6557, +1966-09-01,3.7,0,,33.8,34.9789, +1966-10-01,3.7,0,,34.0,35.2212, +1966-11-01,3.6,0,88.3,34.0,34.9788, +1966-12-01,3.8,0,,34.1,35.0596, +1967-01-01,3.9,0,,34.2,35.225, +1967-02-01,3.8,0,94.1,34.2,34.8255, +1967-03-01,3.8,0,,34.3,34.6292, +1967-04-01,3.8,0,,34.4,34.9558, +1967-05-01,3.8,0,95.9,34.5,34.6508, +1967-06-01,3.9,0,,34.6,34.6465, +1967-07-01,3.8,0,,34.7,34.5676, +1967-08-01,3.8,0,97.0,34.9,35.2299, +1967-09-01,3.8,0,,35.0,35.1727, +1967-10-01,4.0,0,,35.1,35.4591, +1967-11-01,3.9,0,92.9,35.2,35.9663, +1967-12-01,3.8,0,,35.4,36.3536, +1968-01-01,3.7,0,,35.5,36.3145, +1968-02-01,3.8,0,97.2,35.7,36.4446, +1968-03-01,3.7,0,,35.8,36.5584, +1968-04-01,3.5,0,,35.9,36.6113, +1968-05-01,3.5,0,92.4,36.0,37.022, +1968-06-01,3.7,0,,36.2,37.158, +1968-07-01,3.7,0,,36.4,37.101, +1968-08-01,3.5,0,92.4,36.5,37.2045, +1968-09-01,3.4,0,,36.7,37.3454, +1968-10-01,3.4,0,,36.9,37.4196, +1968-11-01,3.4,0,91.7,37.1,37.9036, +1968-12-01,3.4,0,,37.2,38.0233, +1969-01-01,3.4,0,,37.3,38.2533, +1969-02-01,3.4,0,98.2,37.6,38.4981, +1969-03-01,3.4,0,,37.8,38.8, +1969-04-01,3.4,0,,38.1,38.6573, +1969-05-01,3.4,0,91.5,38.1,38.5111, +1969-06-01,3.5,0,,38.3,38.8874, +1969-07-01,3.5,0,,38.5,39.0927, +1969-08-01,3.5,0,86.4,38.7,39.1825, +1969-09-01,3.7,0,,38.9,39.1737, +1969-10-01,3.7,0,,39.1,39.1852, +1969-11-01,3.5,0,79.7,39.2,38.8158, +1969-12-01,3.5,0,,39.4,38.7116, +1970-01-01,3.9,1,,39.6,37.9948, +1970-02-01,4.2,1,78.1,39.8,37.9698, +1970-03-01,4.4,1,,40.1,37.9205, +1970-04-01,4.6,1,,40.4,37.823, +1970-05-01,4.8,1,75.4,40.5,37.7789, +1970-06-01,4.9,1,,40.8,37.6567, +1970-07-01,5.0,1,,40.9,37.7491, +1970-08-01,5.1,1,77.6,41.1,37.6818, +1970-09-01,5.4,1,,41.3,37.4221, +1970-10-01,5.5,1,,41.5,36.6735, +1970-11-01,5.9,1,72.4,41.8,36.4514, +1970-12-01,6.1,0,,42.0,37.2885, +1971-01-01,5.9,0,,42.1,37.5753, +1971-02-01,5.9,0,78.1,42.2,37.5037, +1971-03-01,6.0,0,,42.2,37.4631, +1971-04-01,5.9,0,,42.4,37.6735, +1971-05-01,5.9,0,80.2,42.6,37.8646, +1971-06-01,5.9,0,,42.8,38.0234, +1971-07-01,6.0,0,,42.9,37.9131, +1971-08-01,6.1,0,82.1,43.0,37.693, +1971-09-01,6.0,0,,43.0,38.3056, +1971-10-01,5.8,0,,43.1,38.5924, +1971-11-01,6.0,0,82.0,43.2,38.7561, +1971-12-01,6.0,0,,43.3,39.2035, +1972-01-01,5.8,0,,43.5,40.1408, +1972-02-01,5.7,0,92.8,43.6,40.5459, +1972-03-01,5.8,0,,43.6,40.8263, +1972-04-01,5.7,0,,43.8,41.2348, +1972-05-01,5.7,0,88.6,43.9,41.2405, +1972-06-01,5.7,0,,44.0,41.3497, +1972-07-01,5.6,0,,44.1,41.3035, +1972-08-01,5.6,0,95.2,44.3,41.879, +1972-09-01,5.5,0,,44.3,42.2132, +1972-10-01,5.6,0,,44.4,42.7494, +1972-11-01,5.3,0,90.7,44.4,43.2548, +1972-12-01,5.2,0,,44.6,43.7367, +1973-01-01,4.9,0,,44.6,44.0529, +1973-02-01,5.0,0,81.9,44.8,44.6827, +1973-03-01,4.9,0,,45.0,44.7045, +1973-04-01,5.0,0,,45.1,44.6387, +1973-05-01,4.9,0,77.0,45.3,44.9429, +1973-06-01,4.9,0,,45.4,45.0035, +1973-07-01,4.8,0,,45.5,45.1636, +1973-08-01,4.8,0,72.0,45.7,45.1019, +1973-09-01,4.8,0,,46.0,45.4624, +1973-10-01,4.6,0,,46.3,45.847, +1973-11-01,4.8,0,76.5,46.5,46.0553, +1973-12-01,4.9,1,,46.7,45.923, +1974-01-01,5.1,1,,46.9,45.6204, +1974-02-01,5.2,1,61.8,47.2,45.4759, +1974-03-01,5.1,1,,47.6,45.4777, +1974-04-01,5.1,1,,47.9,45.3451, +1974-05-01,5.1,1,72.1,48.5,45.7044, +1974-06-01,5.4,1,,49.0,45.6238, +1974-07-01,5.5,1,,49.5,45.7221, +1974-08-01,5.5,1,64.4,50.2,45.2366, +1974-09-01,5.9,1,,50.7,45.2365, +1974-10-01,6.0,1,,51.2,45.1133, +1974-11-01,6.6,1,59.5,51.6,43.6278, +1974-12-01,7.2,1,,52.0,42.0533, +1975-01-01,8.1,1,,52.3,41.4676, +1975-02-01,8.1,1,57.6,52.8,40.5165, +1975-03-01,8.6,1,,53.0,40.0638, +1975-04-01,8.8,0,,53.3,40.1573, +1975-05-01,9.0,0,72.8,53.5,40.0253, +1975-06-01,8.8,0,,53.8,40.2772, +1975-07-01,8.6,0,,54.0,40.6994, +1975-08-01,8.4,0,75.7,54.2,41.0998, +1975-09-01,8.4,0,,54.5,41.6304, +1975-10-01,8.4,0,,54.8,41.8135, +1975-11-01,8.3,0,75.6,55.2,41.8697, +1975-12-01,8.2,0,,55.5,42.4414, +1976-01-01,7.9,0,,55.9,43.0012, +1976-02-01,7.7,0,84.6,56.2,43.4229, +1976-03-01,7.6,0,,56.5,43.5168, +1976-04-01,7.7,0,,56.7,43.7327, +1976-05-01,7.4,0,83.3,57.0,43.9186, +1976-06-01,7.6,0,,57.2,43.9637, +1976-07-01,7.8,0,,57.6,44.2141, +1976-08-01,7.8,0,89.7,57.9,44.5182, +1976-09-01,7.6,0,,58.2,44.6721, +1976-10-01,7.7,0,,58.5,44.6585, +1976-11-01,7.8,0,87.0,58.7,45.3392, +1976-12-01,7.8,0,,58.9,45.8294, +1977-01-01,7.5,0,,59.3,45.5087, +1977-02-01,7.6,0,87.1,59.7,46.2277, +1977-03-01,7.4,0,,60.0,46.8185, +1977-04-01,7.2,0,,60.3,47.2684, +1977-05-01,7.0,0,90.2,60.6,47.6411, +1977-06-01,7.2,0,,61.0,48.0116, +1977-07-01,6.9,0,,61.2,48.073, +1977-08-01,7.0,0,89.0,61.5,48.1205, +1977-09-01,6.8,0,,61.8,48.3189, +1977-10-01,6.8,0,,62.0,48.3871, +1977-11-01,6.8,0,84.4,62.3,48.4761, +1977-12-01,6.4,0,,62.7,48.5629, +1978-01-01,6.4,0,83.7,63.1,47.8481, +1978-02-01,6.3,0,84.3,63.4,48.1044, +1978-03-01,6.3,0,78.8,63.8,49.0265, +1978-04-01,6.1,0,81.6,64.3,50.0206, +1978-05-01,6.0,0,82.9,64.7,50.254, +1978-06-01,5.9,0,80.0,65.2,50.5801, +1978-07-01,6.2,0,82.4,65.6,50.5384, +1978-08-01,5.9,0,78.4,66.1,50.7458, +1978-09-01,6.0,0,80.4,66.7,50.8744, +1978-10-01,5.8,0,79.3,67.2,51.2559, +1978-11-01,5.9,0,75.0,67.6,51.6514, +1978-12-01,6.0,0,66.1,68.0,51.909, +1979-01-01,5.9,0,72.1,68.5,51.6278, +1979-02-01,5.9,0,73.9,69.2,51.8752, +1979-03-01,5.8,0,68.4,69.8,52.0446, +1979-04-01,5.8,0,66.0,70.3,51.4193, +1979-05-01,5.6,0,68.1,70.8,51.8856, +1979-06-01,5.7,0,65.8,71.3,51.8965, +1979-07-01,5.7,0,60.4,71.9,51.8097, +1979-08-01,6.0,0,64.5,72.7,51.4684, +1979-09-01,5.9,0,66.7,73.3,51.4597, +1979-10-01,6.0,0,62.1,74.0,51.8334, +1979-11-01,5.9,0,63.3,74.8,51.7531, +1979-12-01,6.0,0,61.0,75.7,51.7878, +1980-01-01,6.3,0,67.0,76.7,52.0679, +1980-02-01,6.3,1,66.9,77.5,52.082, +1980-03-01,6.3,1,56.5,78.6,51.87, +1980-04-01,6.9,1,52.7,79.5,50.8397, +1980-05-01,7.5,1,51.7,80.1,49.589, +1980-06-01,7.6,1,58.7,81.0,48.9482, +1980-07-01,7.8,1,62.3,80.8,48.6039, +1980-08-01,7.7,0,67.3,81.3,48.7563, +1980-09-01,7.5,0,73.7,82.1,49.5608, +1980-10-01,7.5,0,75.0,83.0,50.1908, +1980-11-01,7.5,0,76.7,83.9,51.0311, +1980-12-01,7.2,0,64.5,84.9,51.3708, +1981-01-01,7.5,0,71.4,85.4,51.0415, +1981-02-01,7.4,0,66.9,85.9,50.824, +1981-03-01,7.4,0,66.5,86.4,51.0826, +1981-04-01,7.2,0,72.4,87.0,50.858, +1981-05-01,7.5,0,76.3,87.8,51.1495, +1981-06-01,7.5,0,73.1,88.6,51.4101, +1981-07-01,7.2,0,74.1,89.8,51.7471, +1981-08-01,7.4,1,77.2,90.7,51.6826, +1981-09-01,7.6,1,73.1,91.8,51.4337, +1981-10-01,7.9,1,70.3,92.1,51.0281, +1981-11-01,8.3,1,62.5,92.5,50.435, +1981-12-01,8.5,1,64.3,93.0,49.8987, +1982-01-01,8.6,1,71.0,93.3,48.8685, +1982-02-01,8.9,1,66.5,93.8,49.8636, +1982-03-01,9.0,1,62.0,93.9,49.5323, +1982-04-01,9.3,1,65.5,94.7,49.075, +1982-05-01,9.4,1,67.5,95.4,48.7483, +1982-06-01,9.6,1,65.7,96.1,48.6214, +1982-07-01,9.8,1,65.4,96.7,48.4616, +1982-08-01,9.8,1,65.4,97.1,48.0192, +1982-09-01,10.1,1,69.3,97.2,47.9008, +1982-10-01,10.4,1,73.4,97.5,47.4371, +1982-11-01,10.8,1,72.1,97.3,47.283, +1982-12-01,10.8,0,71.9,97.2,46.9485, +1983-01-01,10.4,0,70.4,97.6,47.8234, +1983-02-01,10.4,0,74.6,98.0,47.5233, +1983-03-01,10.3,0,80.8,98.2,47.9244, +1983-04-01,10.2,0,89.1,98.6,48.5311, +1983-05-01,10.1,0,93.3,98.9,48.8239, +1983-06-01,10.1,0,92.2,99.2,49.123, +1983-07-01,9.4,0,92.8,99.8,49.8546, +1983-08-01,9.5,0,90.9,100.1,50.4443, +1983-09-01,9.2,0,89.9,100.5,51.178, +1983-10-01,8.8,0,89.3,101.0,51.5693, +1983-11-01,8.5,0,91.1,101.5,51.7993, +1983-12-01,8.3,0,94.2,101.8,52.0594, +1984-01-01,8.0,0,100.1,102.5,53.0985, +1984-02-01,7.8,0,97.4,102.8,53.3318, +1984-03-01,7.8,0,101.0,103.2,53.5929, +1984-04-01,7.7,0,96.1,103.7,53.91, +1984-05-01,7.4,0,98.1,104.1,54.2026, +1984-06-01,7.2,0,95.5,104.5,54.4014, +1984-07-01,7.5,0,96.6,105.0,54.5521, +1984-08-01,7.5,0,99.1,105.4,54.6083, +1984-09-01,7.3,0,100.9,105.8,54.4466, +1984-10-01,7.4,0,96.3,106.2,54.4406, +1984-11-01,7.2,0,95.7,106.4,54.624, +1984-12-01,7.3,0,92.9,106.8,54.6416, +1985-01-01,7.3,0,96.0,107.1,54.6322, +1985-02-01,7.2,0,93.7,107.7,54.7825, +1985-03-01,7.2,0,93.7,108.1,54.8653, +1985-04-01,7.3,0,94.6,108.4,54.7459, +1985-05-01,7.2,0,91.8,108.8,54.8255, +1985-06-01,7.4,0,96.5,109.1,54.7989, +1985-07-01,7.4,0,94.0,109.4,54.5396, +1985-08-01,7.1,0,92.4,109.8,54.7055, +1985-09-01,7.1,0,92.1,110.0,54.8906, +1985-10-01,7.1,0,88.4,110.5,54.7199, +1985-11-01,7.0,0,90.9,111.1,54.9124, +1985-12-01,7.0,0,93.9,111.4,55.4412, +1986-01-01,6.7,0,95.6,111.9,55.7481, +1986-02-01,7.2,0,95.9,112.2,55.3568, +1986-03-01,7.2,0,95.1,112.5,54.9625, +1986-04-01,7.1,0,96.2,112.9,55.0413, +1986-05-01,7.2,0,94.8,113.1,55.1226, +1986-06-01,7.2,0,99.3,113.4,54.9005, +1986-07-01,7.0,0,97.7,113.8,55.2592, +1986-08-01,6.9,0,94.9,114.2,55.1567, +1986-09-01,7.0,0,91.9,114.6,55.2975, +1986-10-01,7.0,0,95.6,115.0,55.5332, +1986-11-01,6.9,0,91.4,115.3,55.7557, +1986-12-01,6.6,0,89.1,115.6,56.2845, +1987-01-01,6.6,0,90.4,115.9,56.0882, +1987-02-01,6.6,0,90.2,116.2,56.8275, +1987-03-01,6.6,0,90.8,116.6,56.9098, +1987-04-01,6.3,0,92.8,117.3,57.2701, +1987-05-01,6.3,0,91.1,117.7,57.6322, +1987-06-01,6.2,0,91.5,117.9,57.9307, +1987-07-01,6.1,0,93.7,118.3,58.3258, +1987-08-01,6.0,0,94.4,118.7,58.7636, +1987-09-01,5.9,0,93.6,119.2,58.9951, +1987-10-01,6.0,0,89.3,119.8,59.781, +1987-11-01,5.8,0,83.1,120.1,60.0726, +1987-12-01,5.7,0,86.8,120.4,60.4095, +1988-01-01,5.7,0,90.8,120.9,60.3976, +1988-02-01,5.7,0,91.6,121.2,60.7003, +1988-03-01,5.7,0,94.6,121.7,60.8501, +1988-04-01,5.4,0,91.2,122.3,61.2031, +1988-05-01,5.6,0,94.8,122.7,61.0797, +1988-06-01,5.4,0,94.7,123.2,61.2569, +1988-07-01,5.4,0,93.4,123.6,61.2659, +1988-08-01,5.6,0,97.4,124.0,61.5731, +1988-09-01,5.4,0,97.3,124.7,61.3807, +1988-10-01,5.4,0,94.1,125.2,61.6453, +1988-11-01,5.3,0,93.0,125.6,61.7944, +1988-12-01,5.3,0,91.9,126.0,62.0743, +1989-01-01,5.4,0,97.9,126.5,62.2574, +1989-02-01,5.2,0,95.4,126.9,61.9819, +1989-03-01,5.0,0,94.3,127.4,62.1369, +1989-04-01,5.2,0,91.5,127.8,62.1276, +1989-05-01,5.2,0,90.7,128.3,61.7744, +1989-06-01,5.3,0,90.6,128.8,61.7697, +1989-07-01,5.2,0,92.0,129.2,61.1738, +1989-08-01,5.2,0,89.6,129.5,61.7732, +1989-09-01,5.3,0,95.8,129.9,61.5796, +1989-10-01,5.3,0,93.9,130.6,61.4966, +1989-11-01,5.4,0,90.9,131.1,61.7044, +1989-12-01,5.4,0,90.5,131.6,62.0428, +1990-01-01,5.4,0,93.0,132.1,61.729, +1990-02-01,5.3,0,89.5,132.7,62.2896, +1990-03-01,5.2,0,91.3,133.5,62.5999, +1990-04-01,5.4,0,93.9,134.0,62.4359, +1990-05-01,5.4,0,90.6,134.4,62.6258, +1990-06-01,5.2,0,88.3,135.1,62.8382, +1990-07-01,5.5,0,88.2,135.8,62.7284, +1990-08-01,5.7,1,76.4,136.6,62.9475, +1990-09-01,5.9,1,72.8,137.1,62.9523, +1990-10-01,5.9,1,63.9,137.6,62.5845, +1990-11-01,6.2,1,66.0,138.0,61.7887, +1990-12-01,6.3,1,65.5,138.6,61.3352, +1991-01-01,6.4,1,66.8,139.5,61.1355, +1991-02-01,6.6,1,70.4,140.2,60.6838, +1991-03-01,6.8,1,87.7,140.5,60.3346, +1991-04-01,6.7,0,81.8,140.9,60.4938, +1991-05-01,6.9,0,78.3,141.3,61.0633, +1991-06-01,6.9,0,82.1,141.8,61.5581, +1991-07-01,6.8,0,82.9,142.3,61.7305, +1991-08-01,6.9,0,82.0,142.9,61.7298, +1991-09-01,6.9,0,83.0,143.4,62.2195, +1991-10-01,7.0,0,78.3,143.7,62.1855, +1991-11-01,7.0,0,69.1,144.2,62.0871, +1991-12-01,7.3,0,68.2,144.7,61.7852, +1992-01-01,7.3,0,67.5,145.1,61.4616, +1992-02-01,7.4,0,68.8,145.4,61.8955, +1992-03-01,7.4,0,76.0,145.9,62.4232, +1992-04-01,7.4,0,77.2,146.3,62.897, +1992-05-01,7.6,0,79.2,146.8,63.104, +1992-06-01,7.8,0,80.4,147.1,63.1363, +1992-07-01,7.7,0,76.6,147.6,63.7176, +1992-08-01,7.6,0,76.1,147.9,63.3613, +1992-09-01,7.6,0,75.6,148.1,63.5703, +1992-10-01,7.3,0,73.3,148.8,63.9995, +1992-11-01,7.4,0,85.3,149.2,64.2422, +1992-12-01,7.4,0,91.0,149.6,64.3507, +1993-01-01,7.3,0,89.3,150.1,64.5978, +1993-02-01,7.1,0,86.6,150.6,64.9089, +1993-03-01,7.0,0,85.9,150.8,64.8522, +1993-04-01,7.1,0,85.6,151.4,65.0301, +1993-05-01,7.1,0,80.3,151.8,64.784, +1993-06-01,7.0,0,81.5,152.1,64.9301, +1993-07-01,6.9,0,77.0,152.3,65.1052, +1993-08-01,6.8,0,77.3,152.8,65.0267, +1993-09-01,6.7,0,77.9,152.9,65.3717, +1993-10-01,6.8,0,82.7,153.4,65.8509, +1993-11-01,6.6,0,81.2,153.9,66.1518, +1993-12-01,6.5,0,88.2,154.3,66.5436, +1994-01-01,6.6,0,94.3,154.5,66.7351, +1994-02-01,6.6,0,93.2,154.8,66.774, +1994-03-01,6.5,0,91.5,155.3,67.4552, +1994-04-01,6.4,0,92.6,155.5,67.8541, +1994-05-01,6.1,0,92.8,155.9,68.1663, +1994-06-01,6.1,0,91.2,156.4,68.6265, +1994-07-01,6.1,0,89.0,156.7,68.7178, +1994-08-01,6.0,0,91.7,157.1,69.1785, +1994-09-01,5.9,0,91.5,157.5,69.3961, +1994-10-01,5.8,0,92.7,157.8,69.9544, +1994-11-01,5.6,0,91.6,158.2,70.4326, +1994-12-01,5.5,0,95.1,158.3,71.1523, +1995-01-01,5.6,0,97.6,159.0,71.2734, +1995-02-01,5.4,0,95.1,159.4,71.193, +1995-03-01,5.4,0,90.3,159.9,71.3113, +1995-04-01,5.8,0,92.5,160.4,71.1882, +1995-05-01,5.6,0,89.8,160.7,71.5378, +1995-06-01,5.6,0,92.7,161.1,71.7749, +1995-07-01,5.7,0,94.4,161.4,71.4748, +1995-08-01,5.7,0,96.2,161.8,72.4223, +1995-09-01,5.6,0,88.9,162.2,72.7144, +1995-10-01,5.5,0,90.2,162.7,72.5866, +1995-11-01,5.6,0,88.2,163.0,72.7779, +1995-12-01,5.6,0,91.0,163.1,73.0305, +1996-01-01,5.6,0,89.3,163.7,72.6125, +1996-02-01,5.5,0,88.5,164.0,73.6752, +1996-03-01,5.5,0,93.7,164.4,73.585, +1996-04-01,5.6,0,92.7,164.6,74.3172, +1996-05-01,5.6,0,89.4,165.0,74.8726, +1996-06-01,5.3,0,92.4,165.4,75.4433, +1996-07-01,5.5,0,94.7,165.7,75.4549, +1996-08-01,5.1,0,95.3,166.0,75.8411, +1996-09-01,5.2,0,94.7,166.5,76.3158, +1996-10-01,5.2,0,96.5,166.8,76.3384, +1996-11-01,5.4,0,99.2,167.2,77.0397, +1996-12-01,5.4,0,96.9,167.4,77.5333, +1997-01-01,5.3,0,97.4,167.8,77.6562, +1997-02-01,5.2,0,99.7,168.1,78.5833, +1997-03-01,5.2,0,100.0,168.4,79.082, +1997-04-01,5.1,0,101.4,168.9,79.1747, +1997-05-01,4.9,0,103.2,169.2,79.6133, +1997-06-01,5.0,0,104.5,169.4,79.9792, +1997-07-01,4.9,0,107.1,169.7,80.6624, +1997-08-01,4.8,0,104.4,169.8,81.4694, +1997-09-01,4.9,0,106.0,170.2,82.2156, +1997-10-01,4.7,0,105.6,170.6,82.9446, +1997-11-01,4.6,0,107.2,170.8,83.5978, +1997-12-01,4.7,0,102.1,171.2,83.9538, +1998-01-01,4.6,0,106.6,171.6,84.3218, +1998-02-01,4.6,0,110.4,171.9,84.4583, +1998-03-01,4.7,0,106.5,172.2,84.488, +1998-04-01,4.3,0,108.7,172.5,84.7779, +1998-05-01,4.4,0,106.5,172.9,85.3367, +1998-06-01,4.5,0,105.6,173.2,84.8248, +1998-07-01,4.5,0,105.2,173.5,84.4986, +1998-08-01,4.5,0,104.4,174.0,86.248, +1998-09-01,4.6,0,100.9,174.2,86.2021, +1998-10-01,4.5,0,97.4,174.4,86.7914, +1998-11-01,4.4,0,102.7,174.8,86.7256, +1998-12-01,4.4,0,100.5,175.4,87.1185, +1999-01-01,4.3,0,103.9,175.6,87.4949, +1999-02-01,4.4,0,108.1,175.6,88.027, +1999-03-01,4.2,0,105.7,175.7,88.2267, +1999-04-01,4.3,0,104.6,176.3,88.4295, +1999-05-01,4.2,0,106.8,176.5,88.9886, +1999-06-01,4.3,0,107.3,176.6,88.9188, +1999-07-01,4.3,0,106.0,177.1,89.4333, +1999-08-01,4.2,0,104.5,177.3,89.7924, +1999-09-01,4.2,0,107.2,177.8,89.4299, +1999-10-01,4.1,0,103.2,178.1,90.523, +1999-11-01,4.1,0,107.2,178.4,90.9618, +1999-12-01,4.0,0,105.4,178.7,91.6634, +2000-01-01,4.0,0,112.0,179.3,91.538, +2000-02-01,4.1,0,111.3,179.4,91.8239, +2000-03-01,4.0,0,107.1,180.0,92.1504, +2000-04-01,3.8,0,109.2,180.3,92.6989, +2000-05-01,4.0,0,110.7,180.7,92.9499, +2000-06-01,4.0,0,106.4,181.1,92.9862, +2000-07-01,4.0,0,108.3,181.5,92.8048, +2000-08-01,4.1,0,107.3,181.9,92.5521, +2000-09-01,3.9,0,106.8,182.3,92.9792, +2000-10-01,3.9,0,105.8,182.6,92.6179, +2000-11-01,3.9,0,107.6,183.1,92.6438, +2000-12-01,3.9,0,98.4,183.3,92.3415, +2001-01-01,4.2,0,94.7,183.9,91.902, +2001-02-01,4.2,0,90.6,184.4,91.3034, +2001-03-01,4.3,0,91.5,184.7,91.1162, +2001-04-01,4.4,1,88.4,185.1,90.7891, +2001-05-01,4.3,1,92.0,185.3,90.3555, +2001-06-01,4.5,1,92.6,186.0,89.8784, +2001-07-01,4.6,1,92.4,186.4,89.3028, +2001-08-01,4.9,1,91.5,186.7,89.2003, +2001-09-01,5.0,1,81.8,187.1,88.7485, +2001-10-01,5.3,1,82.7,187.4,88.5052, +2001-11-01,5.5,1,83.9,188.1,87.97, +2001-12-01,5.7,0,88.8,188.4,87.9324, +2002-01-01,5.7,0,93.0,188.7,88.5176, +2002-02-01,5.7,0,90.7,189.1,88.5278, +2002-03-01,5.7,0,95.7,189.2,89.1686, +2002-04-01,5.9,0,93.0,189.7,89.604, +2002-05-01,5.8,0,96.9,190.0,89.964, +2002-06-01,5.8,0,92.4,190.2,90.7061, +2002-07-01,5.8,0,88.1,190.5,90.6769, +2002-08-01,5.7,0,87.6,191.1,90.5762, +2002-09-01,5.7,0,86.1,191.3,90.6531, +2002-10-01,5.7,0,80.6,191.5,90.4643, +2002-11-01,5.9,0,84.2,191.9,90.9711, +2002-12-01,6.0,0,86.7,192.1,90.4652, +2003-01-01,5.8,0,82.4,192.4,91.2361, +2003-02-01,5.9,0,79.9,192.5,91.3445, +2003-03-01,5.9,0,77.6,192.5,91.1044, +2003-04-01,6.0,0,86.0,192.5,90.5617, +2003-05-01,6.1,0,92.1,192.9,90.537, +2003-06-01,6.3,0,89.7,193.0,90.6527, +2003-07-01,6.2,0,90.9,193.4,91.1185, +2003-08-01,6.1,0,89.3,193.6,90.9197, +2003-09-01,6.1,0,87.7,193.7,91.5059, +2003-10-01,6.0,0,89.6,194.0,91.6186, +2003-11-01,5.8,0,93.7,194.0,92.2329, +2003-12-01,5.7,0,92.6,194.2,92.2602, +2004-01-01,5.7,0,103.8,194.6,92.4006, +2004-02-01,5.6,0,94.4,194.9,92.958, +2004-03-01,5.8,0,95.8,195.5,92.5861, +2004-04-01,5.6,0,94.2,195.9,92.93, +2004-05-01,5.6,0,90.2,196.2,93.6013, +2004-06-01,5.6,0,95.6,196.6,92.871, +2004-07-01,5.5,0,96.7,196.8,93.5649, +2004-08-01,5.4,0,95.9,196.9,93.6277, +2004-09-01,5.4,0,94.2,197.5,93.7467, +2004-10-01,5.5,0,91.7,197.9,94.5465, +2004-11-01,5.4,0,92.8,198.3,94.7813, +2004-12-01,5.4,0,97.1,198.6,95.5365, +2005-01-01,5.3,0,95.5,199.0,95.8801, +2005-02-01,5.4,0,94.1,199.4,96.5761, +2005-03-01,5.2,0,92.6,200.1,96.4456, +2005-04-01,5.2,0,87.7,200.2,96.6532, +2005-05-01,5.1,0,86.9,200.5,96.7402, +2005-06-01,5.0,0,96.0,200.6,97.1753, +2005-07-01,5.0,0,96.5,200.9,96.8676, +2005-08-01,4.9,0,89.1,201.1,97.1833, +2005-09-01,5.0,0,76.9,201.3,95.3005, +2005-10-01,5.0,0,74.2,202.0,96.4572, +2005-11-01,5.0,0,81.6,202.5,97.5414, +2005-12-01,4.9,0,91.5,202.8,98.0452, +2006-01-01,4.7,0,91.2,203.2,98.1999, +2006-02-01,4.8,0,86.7,203.6,98.2413, +2006-03-01,4.7,0,88.9,204.3,98.4628, +2006-04-01,4.7,0,87.4,204.8,98.7618, +2006-05-01,4.6,0,79.1,205.4,98.7869, +2006-06-01,4.6,0,84.9,205.9,99.0997, +2006-07-01,4.7,0,84.7,206.3,99.0489, +2006-08-01,4.7,0,82.0,206.8,99.4836, +2006-09-01,4.5,0,85.4,207.2,99.3183, +2006-10-01,4.4,0,93.6,207.6,99.2044, +2006-11-01,4.5,0,92.1,207.8,99.1627, +2006-12-01,4.4,0,91.7,208.1,100.1735, +2007-01-01,4.6,0,96.9,208.6,99.8346, +2007-02-01,4.5,0,91.3,209.135,100.7753, +2007-03-01,4.4,0,88.4,209.418,100.9547, +2007-04-01,4.5,0,87.1,209.747,101.6334, +2007-05-01,4.4,0,88.3,210.058,101.7306, +2007-06-01,4.6,0,85.3,210.392,101.7893, +2007-07-01,4.7,0,90.4,210.773,101.6672, +2007-08-01,4.6,0,83.4,211.119,101.863, +2007-09-01,4.7,0,83.4,211.554,102.0894, +2007-10-01,4.7,0,80.9,212.077,101.8015, +2007-11-01,4.7,0,76.1,212.66,102.3446, +2007-12-01,5.0,0,75.5,213.168,102.3803, +2008-01-01,5.0,1,78.4,213.771,102.2448, +2008-02-01,4.9,1,70.8,213.939,101.8438, +2008-03-01,5.1,1,69.5,214.42,101.5336, +2008-04-01,5.0,1,62.6,214.56,100.8785, +2008-05-01,5.4,1,59.8,214.936,100.2674, +2008-06-01,5.6,1,56.4,215.424,100.0335, +2008-07-01,5.8,1,61.2,215.965,99.676, +2008-08-01,6.1,1,63.0,216.393,98.1111, +2008-09-01,6.1,1,70.3,216.713,93.7895, +2008-10-01,6.5,1,57.6,216.788,94.7219, +2008-11-01,6.8,1,55.3,216.947,93.4715, +2008-12-01,7.3,1,60.1,216.925,90.8357, +2009-01-01,7.8,1,61.2,217.346,88.57, +2009-02-01,8.3,1,56.3,217.792,88.031, +2009-03-01,8.7,1,57.3,218.253,86.6528, +2009-04-01,9.0,1,65.1,218.706,85.9344, +2009-05-01,9.4,1,68.7,218.904,85.0665, +2009-06-01,9.5,1,70.8,219.112,84.7459, +2009-07-01,9.5,0,66.0,219.263,85.7027, +2009-08-01,9.6,0,65.7,219.496,86.6426, +2009-09-01,9.8,0,73.5,219.92,87.4063, +2009-10-01,10.0,0,70.6,220.501,87.6419, +2009-11-01,9.9,0,67.4,220.666,88.0221, +2009-12-01,9.9,0,72.5,220.881,88.3793, +2010-01-01,9.8,0,74.4,220.633,89.3426, +2010-02-01,9.8,0,73.6,220.731,89.6779, +2010-03-01,9.9,0,73.6,220.783,90.2928, +2010-04-01,9.9,0,72.2,220.822,90.5991, +2010-05-01,9.6,0,73.6,220.962,91.823, +2010-06-01,9.4,0,76.0,221.194,91.9928, +2010-07-01,9.4,0,67.8,221.363,92.3421, +2010-08-01,9.5,0,68.9,221.509,92.6805, +2010-09-01,9.5,0,68.2,221.711,92.9537, +2010-10-01,9.4,0,67.7,221.83,92.6903, +2010-11-01,9.8,0,71.6,222.149,92.7709, +2010-12-01,9.3,0,74.5,222.343,93.6787, +2011-01-01,9.1,0,74.2,222.803,93.4814, +2011-02-01,9.0,0,77.5,223.213,93.1102, +2011-03-01,9.0,0,67.5,223.454,94.0788, +2011-04-01,9.1,0,69.8,223.727,93.7749, +2011-05-01,9.0,0,74.3,224.175,93.888, +2011-06-01,9.1,0,71.5,224.697,94.172, +2011-07-01,9.0,0,63.7,225.218,94.636, +2011-08-01,9.0,0,55.8,225.862,95.2749, +2011-09-01,9.0,0,59.5,226.118,95.1961, +2011-10-01,8.8,0,60.8,226.506,95.8568, +2011-11-01,8.6,0,63.7,226.899,95.8485, +2011-12-01,8.5,0,69.9,227.405,96.3571, +2012-01-01,8.3,0,75.0,227.877,96.9318, +2012-02-01,8.3,0,75.3,228.034,97.2435, +2012-03-01,8.2,0,76.2,228.478,96.74, +2012-04-01,8.2,0,76.4,228.905,97.4359, +2012-05-01,8.2,0,79.3,229.224,97.6609, +2012-06-01,8.2,0,73.2,229.623,97.6663, +2012-07-01,8.2,0,72.3,229.97,97.8713, +2012-08-01,8.1,0,74.3,230.233,97.4525, +2012-09-01,7.8,0,78.3,230.659,97.381, +2012-10-01,7.8,0,82.6,231.024,97.7193, +2012-11-01,7.7,0,82.7,231.33,98.0777, +2012-12-01,7.9,0,72.9,231.725,98.3101, +2013-01-01,8.0,0,73.8,232.229,98.3342, +2013-02-01,7.7,0,77.6,232.569,98.7941, +2013-03-01,7.5,0,78.6,232.794,99.1921, +2013-04-01,7.6,0,76.4,232.832,99.1065, +2013-05-01,7.5,0,84.5,232.996,99.1839, +2013-06-01,7.5,0,84.1,233.35,99.3897, +2013-07-01,7.3,0,85.1,233.88,99.0814, +2013-08-01,7.2,0,82.1,234.336,99.6543, +2013-09-01,7.2,0,77.5,234.7,100.161, +2013-10-01,7.2,0,73.2,234.921,100.082, +2013-11-01,6.9,0,75.1,235.359,100.3158, +2013-12-01,6.7,0,82.5,235.759,100.5175, +2014-01-01,6.6,0,81.2,235.961,100.1599, +2014-02-01,6.7,0,81.6,236.185,100.9168, +2014-03-01,6.7,0,80.0,236.625,101.8942, +2014-04-01,6.2,0,84.1,237.072,101.9928, +2014-05-01,6.3,0,81.9,237.529,102.3836, +2014-06-01,6.1,0,82.5,237.837,102.735, +2014-07-01,6.2,0,81.8,238.195,102.9632, +2014-08-01,6.1,0,82.5,238.405,102.7688, +2014-09-01,5.9,0,84.6,238.786,103.0641, +2014-10-01,5.7,0,86.9,239.191,103.0764, +2014-11-01,5.8,0,88.8,239.458,103.7126, +2014-12-01,5.6,0,93.6,239.584,103.7044, +2015-01-01,5.7,0,98.1,239.811,102.8905, +2015-02-01,5.5,0,95.4,240.172,102.2335, +2015-03-01,5.4,0,93.0,240.755,101.8914, +2015-04-01,5.4,0,95.9,241.346,101.3355, +2015-05-01,5.6,0,90.7,241.688,100.8883, +2015-06-01,5.3,0,96.1,242.064,100.571, +2015-07-01,5.2,0,93.1,242.565,101.2228, +2015-08-01,5.1,0,91.9,242.817,101.0367, +2015-09-01,5.0,0,87.2,243.316,100.7547, +2015-10-01,5.0,0,90.0,243.768,100.2675, +2015-11-01,5.1,0,91.3,244.241,99.5298, +2015-12-01,5.0,0,92.6,244.547,99.0417, +2016-01-01,4.8,0,92.0,244.955,99.5607, +2016-02-01,4.9,0,91.7,245.51,99.0445, +2016-03-01,5.0,0,91.0,245.913,98.3096, +2016-04-01,5.1,0,89.0,246.551,98.6421, +2016-05-01,4.8,0,94.7,247.137,98.4184, +2016-06-01,4.9,0,93.5,247.54,98.8833, +2016-07-01,4.8,0,90.0,247.829,98.9992, +2016-08-01,4.9,0,89.8,248.423,98.85, +2016-09-01,5.0,0,91.2,248.842,98.7457, +2016-10-01,4.9,0,87.2,249.142,98.7611, +2016-11-01,4.7,0,93.8,249.481,98.3491, +2016-12-01,4.7,0,98.2,249.92,99.0214, +2017-01-01,4.7,0,98.5,250.467,98.7611, +2017-02-01,4.6,0,96.3,250.998,98.3646, +2017-03-01,4.4,0,96.9,250.944,99.0137, +2017-04-01,4.4,0,97.0,251.227,100.0224, +2017-05-01,4.4,0,97.1,251.43,100.1408, +2017-06-01,4.3,0,95.0,251.746,100.3471, +2017-07-01,4.3,0,93.4,251.985,100.1068, +2017-08-01,4.4,0,96.8,252.535,99.6925, +2017-09-01,4.3,0,95.1,252.812,99.7954, +2017-10-01,4.2,0,100.7,253.526,101.0247, +2017-11-01,4.2,0,98.5,253.816,101.2664, +2017-12-01,4.1,0,95.9,254.344,101.4644, +2018-01-01,4.0,0,95.7,255.204,101.4625, +2018-02-01,4.1,0,99.7,255.711,101.7085, +2018-03-01,4.0,0,101.4,256.271,102.2067, +2018-04-01,4.0,0,98.8,256.63,103.3329, +2018-05-01,3.8,0,98.0,257.145,102.4024, +2018-06-01,4.0,0,98.2,257.399,103.198, +2018-07-01,3.8,0,97.9,257.699,103.3646, +2018-08-01,3.8,0,96.2,257.891,104.0327, +2018-09-01,3.7,0,100.1,258.368,104.1004, +2018-10-01,3.8,0,98.6,258.917,103.9856, +2018-11-01,3.8,0,97.5,259.439,104.0681, +2018-12-01,3.9,0,98.3,260.063,104.0936, +2019-01-01,4.0,0,91.2,260.766,103.4021, +2019-02-01,3.8,0,93.8,261.186,102.8371, +2019-03-01,3.8,0,98.4,261.567,102.8757, +2019-04-01,3.7,0,97.2,261.997,102.274, +2019-05-01,3.6,0,100.0,262.217,102.392, +2019-06-01,3.6,0,98.2,262.739,102.4375, +2019-07-01,3.7,0,98.4,263.28,101.9408, +2019-08-01,3.6,0,89.8,263.877,102.639, +2019-09-01,3.5,0,93.2,264.388,102.2917, +2019-10-01,3.6,0,95.5,264.97,101.4281, +2019-11-01,3.6,0,96.8,265.548,101.9372, +2019-12-01,3.6,0,99.3,266.02,101.7003, +2020-01-01,3.6,0,99.8,266.716,101.0338, +2020-02-01,3.5,0,101.0,267.37,101.3735, +2020-03-01,4.4,1,89.1,267.054,97.4078, +2020-04-01,14.8,1,71.8,265.746,84.5619, +2020-05-01,13.2,0,72.3,265.412,85.9604, +2020-06-01,11.0,0,78.1,265.849,91.5934, +2020-07-01,10.2,0,72.5,267.389,95.0256, +2020-08-01,8.4,0,74.1,268.422,95.9549, +2020-09-01,7.8,0,80.4,268.933,95.9669, +2020-10-01,6.9,0,81.8,269.306,96.7394, +2020-11-01,6.7,0,76.9,269.981,97.0816, +2020-12-01,6.7,0,80.7,270.341,98.363, +2021-01-01,6.4,0,79.0,270.387,98.8822, +2021-02-01,6.2,0,76.8,270.793,95.6149, +2021-03-01,6.1,0,84.9,271.459,98.3856, +2021-04-01,6.1,0,88.3,273.66,98.5587, +2021-05-01,5.8,0,82.9,275.465,99.4333, +2021-06-01,5.9,0,85.5,277.651,99.7987, +2021-07-01,5.4,0,81.2,278.648,100.2519, +2021-08-01,5.1,0,70.3,278.997,100.0437, +2021-09-01,4.7,0,72.8,279.698,98.8579, +2021-10-01,4.5,0,71.7,281.668,100.1861, +2021-11-01,4.1,0,67.4,283.406,100.8654, +2021-12-01,3.9,0,70.6,285.18,100.5726, +2022-01-01,4.0,0,67.2,286.757,100.1856, +2022-02-01,3.9,0,62.8,288.273,100.8064, +2022-03-01,3.7,0,59.4,289.056,101.3907, +2022-04-01,3.7,0,65.2,290.514,101.44, +2022-05-01,3.6,0,58.4,292.063,101.3335, +2022-06-01,3.6,0,50.0,294.05,101.018, +2022-07-01,3.5,0,51.5,295.088,101.223, +2022-08-01,3.6,0,58.2,296.535,101.0963, +2022-09-01,3.5,0,58.6,298.226,101.2918, +2022-10-01,3.6,0,59.9,299.399,101.2529, +2022-11-01,3.6,0,56.7,300.325,100.9552, +2022-12-01,3.5,0,59.8,301.394,99.7664, +2023-01-01,3.5,0,64.9,302.642,100.505, +2023-02-01,3.6,0,66.9,304.123,100.6416, +2023-03-01,3.5,0,62.0,305.147,101.0205, +2023-04-01,3.4,0,63.7,306.534,101.253, +2023-05-01,3.6,0,59.0,307.652,100.9286, +2023-06-01,3.6,0,64.2,308.322,100.1196, +2023-07-01,3.5,0,71.5,308.96,100.9067, +2023-08-01,3.7,0,69.4,309.58,100.8393, +2023-09-01,3.7,0,67.8,310.539,101.0211, +2023-10-01,3.9,0,63.8,311.457,100.4689, +2023-11-01,3.7,0,61.3,312.392,100.8639, +2023-12-01,3.8,0,69.7,313.195,100.6031, +2024-01-01,3.7,0,79.0,314.319,99.2223, +2024-02-01,3.9,0,76.9,315.564,100.285, +2024-03-01,3.9,0,79.4,316.792,100.4575, +2024-04-01,3.9,0,77.2,317.649,100.2434, +2024-05-01,3.9,0,69.1,318.095,100.863, +2024-06-01,4.1,0,68.2,318.39,100.894, +2024-07-01,4.2,0,66.4,318.94,99.9757, +2024-08-01,4.2,0,67.9,319.75,100.4309, +2024-09-01,4.1,0,70.1,320.732,99.8084, +2024-10-01,4.1,0,70.5,321.731,99.4695, +2024-11-01,4.2,0,71.8,322.657,99.2925, +2024-12-01,4.1,0,74.0,323.259,100.3273, +2025-01-01,4.0,0,71.7,324.638,100.0647, +2025-02-01,4.2,0,64.7,325.465,101.0993, +2025-03-01,4.2,0,57.0,325.69,101.0404, +2025-04-01,4.2,0,52.2,326.467,101.1279, +2025-05-01,4.3,0,52.2,326.893,100.9655, +2025-06-01,4.1,0,60.7,327.658,101.4785, +2025-07-01,4.3,0,61.7,328.682,101.894, +2025-08-01,4.3,0,58.2,329.7,101.6247, +2025-09-01,4.4,0,55.1,330.418,101.668, +2025-10-01,,0,53.6,,101.2195, +2025-11-01,4.5,0,51.0,331.043,101.0344, +2025-12-01,4.4,0,52.9,331.814,101.4941, +2026-01-01,4.3,0,56.4,332.793,101.0388, +2026-02-01,4.4,0,56.6,333.512,101.9053, +2026-03-01,4.3,0,53.3,334.165,101.7543, +2026-04-01,4.3,0,49.8,335.423,102.5198, +2026-05-01,4.3,0,44.8,336.121,102.5099, +2026-06-01,4.2,0,49.5,336.065,102.7868, +2026-07-01,4.1,0,55.2,336.789,102.9939, diff --git a/lectures/us_business_cycle_monthly.csv.yml b/lectures/us_business_cycle_monthly.csv.yml new file mode 100644 index 0000000..726d0da --- /dev/null +++ b/lectures/us_business_cycle_monthly.csv.yml @@ -0,0 +1,131 @@ +# Manifest for us_business_cycle_monthly.csv — the FRED half of the intro +# `business_cycle` lecture's data as one composite monthly file, built by +# builders/business_cycle_fred.py on the shared builders/_fred.py library. +# The FILENAME IS PROVISIONAL: it follows the strawman on the naming-policy +# issue (QuantEcon/data-lectures#113 — name a composite for its topic, not +# its lecture) and is free to change while `consumers` is empty. Landed +# 2026-09-01 as part of the P4 dynamic-snapshot pilot; it is the first +# composite dynamic snapshot and the first file with a monthly cadence. + +filename: us_business_cycle_monthly.csv +title: US business-cycle indicators, monthly — unemployment, NBER recessions, consumer sentiment, core CPI, industrial production, 1919 onward +description: > + Six FRED series on one monthly DATE index from 1919-01 to the newest month + FRED carries: the civilian unemployment rate (UNRATE), the NBER recession + indicator (USREC), the Michigan consumer sentiment index (UMCSENT), the CPI + less food and energy (CPILFESL), the industrial production index (INDPRO) + and the 1929–1942 unemployment rate from the NBER Macrohistory database + (M0892AUSM156SNBR), which the lecture splices before UNRATE. One file + rather than six because the lecture reads them together and a single URL + with these column names is what keeps the lecture readable. A consumer + selects series and dates in pandas; every value is in the units FRED + publishes. + +# Constructed, tracking moving sources: FRED appends a month every month and +# revises CPILFESL and INDPRO, so the file is refreshed in place monthly. +class: dynamic-snapshot + +source: + name: FRED (Federal Reserve Bank of St. Louis) — series UNRATE, USREC, UMCSENT, CPILFESL, INDPRO, M0892AUSM156SNBR + url: https://fred.stlouisfed.org + doi: null + version: > + Live FRED at each refresh, one `fredgraph.csv` request per series; the + vintage is pinned by `retrieved` and integrity.upstream. + series: > + UNRATE — civilian unemployment rate, %, SA, monthly from 1948-01 (BLS, + Current Population Survey). USREC — NBER-based recession indicator, 0/1, + from the NBER's peak/trough chronology. UMCSENT — University of Michigan + index of consumer sentiment, 1966Q1=100, quarterly-to-irregular from + 1952-11 and monthly from 1978-01. CPILFESL — CPI for all urban consumers, + all items less food and energy, index 1982-84=100, SA, from 1957-01 + (BLS). INDPRO — industrial production: total index, 2017=100, SA, from + 1919-01 (Board of Governors). M0892AUSM156SNBR — unemployment rate for + the United States, %, NSA, 1929-04 to 1942-06 (NBER Macrohistory, + chapter 8). + citation: > + U.S. Bureau of Labor Statistics (UNRATE, CPILFESL); Federal Reserve Bank + of St. Louis, NBER based Recession Indicators (USREC); University of + Michigan, Surveys of Consumers (UMCSENT); Board of Governors of the + Federal Reserve System, Industrial Production and Capacity Utilization + (INDPRO); National Bureau of Economic Research, Macrohistory Database + (M0892AUSM156SNBR). All retrieved from FRED, Federal Reserve Bank of St. + Louis. + note: > + fredgraph.csv titles its date column `observation_date`; the library + renames it to `DATE`, the header every other FRED file here carries. + UNRATE and CPILFESL are both EMPTY for 2025-10 — the month the 2025 + federal shutdown stopped BLS publishing — and the builder declares that + hole exactly, so a second shutdown month fails the refresh for a human + to look at rather than shipping silently. + +license: + name: null + url: https://fred.stlouisfed.org/legal/ + redistribution: permitted + verified: 2026-09-01 + note: > + Split provenance, one answer. FRED classifies each series on its page + (checked 2026-09-01): UNRATE is "public domain: citation requested", and + CPILFESL, INDPRO and USREC are US federal statistics or built from the + NBER's published dates (the fred_data.csv manifest's 2026-08-17 check of + federal series pages against FRED's S&P 500 page as a positive control is + the precedent). UMCSENT is the one third-party series: FRED classifies it + "copyrighted: citation required" (fred.stlouisfed.org/legal/#copyright- + citation-required) — the University of Michigan's terms permit use with + citation, which is exactly what this repo's cache-with-attribution policy + provides; the manifest's `source.citation` names the Surveys of + Consumers. M0892AUSM156SNBR is NBER Macrohistory data the NBER publishes + openly. So `permitted`, with the Michigan attribution as the condition + that makes it so; if Michigan's terms are ever read as forbidding a + cached CSV, the alternative is to drop UMCSENT from the composite. + +# Stamped by scripts/snapshots.py on every merged refresh. +retrieved: 2026-09-01 +maintainer: QuantEcon + +integrity: + sha256: e6f8e0dea7999246f3bf23b4410dac14e70bac475351560a1f13895132e45f72 + upstream: + # Stamped by scripts/snapshots.py on every merged refresh: a fresh + # snapshot is `verified` by construction. + status: verified + date: 2026-09-01 + against: builders/business_cycle_fred.py + note: "Refreshed 2026-09-01 by the scheduled refresh: these bytes are the builder's validated output from the live source that day, so they are verified by construction. Overlap window n/a: 0 of 0 cells revised, max |change| 0." + +schema: + format: csv + columns: + - {name: DATE, dtype: datetime, description: "first-of-month date stamp on an unbroken monthly grid from 1919-01-01 to the newest month (`schema.date_range.end`); the CSV index column — read it with parse_dates and index_col"} + - {name: UNRATE, dtype: float64, description: "civilian unemployment rate, %, SA; empty before 1948-01 and for 2025-10; validate() bounds it in [0, 30]"} + - {name: USREC, dtype: int64, description: "NBER recession indicator, 1 in recession months and 0 otherwise; complete from 1919-01"} + - {name: UMCSENT, dtype: float64, description: "Michigan consumer sentiment index, 1966Q1=100; empty before 1952-11, sparse (quarterly then irregular) until 1978-01, monthly since; validate() bounds it in [20, 150]"} + - {name: CPILFESL, dtype: float64, description: "CPI less food and energy, index 1982-84=100, SA; empty before 1957-01 and for 2025-10; validate() bounds it in [20, 1000]"} + - {name: INDPRO, dtype: float64, description: "industrial production index, 2017=100, SA; complete from 1919-01; validate() bounds it in [1, 300]"} + - {name: M0892AUSM156SNBR, dtype: float64, description: "unemployment rate, %, NSA, from the NBER Macrohistory database; populated 1929-04 to 1942-06 only; validate() bounds it in [0, 40]"} + row_count_floor: 1291 # 1919-01 .. 2026-07 in the first vintage; + # grows by one row per month. validate() + # asserts the grid and recency (newest month + # at most three months old), not this count. + # `end` is stamped by scripts/snapshots.py on every refresh. + date_range: {start: 1919-01-01, end: 2026-07-01} + # Every null is structural — before a series' first observation, in + # UMCSENT's sparse pre-1978 years, after the historical series' last month, + # or the 2025-10 shutdown hole in UNRATE and CPILFESL — and validate() + # asserts their PLACEMENT exactly. The totals below are the first vintage's + # (they grow only for M0892AUSM156SNBR, by one per month) and are recorded + # for the reader, not asserted. + known_nulls: {UNRATE: 349, UMCSENT: 616, CPILFESL: 457, M0892AUSM156SNBR: 1132} + +# None yet. Decided 2026-09-01: lecture-python-intro keeps its live +# pandas_datareader calls (the API is the lesson there); lecture-wasm will +# read this file once business_cycle is re-enabled in its build +# (QuantEcon/lecture-wasm#70) — record it here with `on_refresh: rebuild`. +# The python.myst unemployment_linear and unemployment_shocks lectures read +# UNRATE live over the same window and are candidates for a later adoption. +consumers: [] + +builder: builders/business_cycle_fred.py +builder_status: committed +cadence: monthly diff --git a/migration.yml b/migration.yml index 0cc1689..ffd0aba 100644 --- a/migration.yml +++ b/migration.yml @@ -848,6 +848,39 @@ datasets: repoints: [] cutover: null + # The rest of the business_cycle set (#114): two more World Bank tables from + # the same builder, and the FRED half as one composite monthly file. Names + # provisional pending #113. Consumer-to-be: lecture-wasm (lecture-wasm#70). + unemployment_rate_annual.csv: + pilot: P4 + status: landed + prior_pattern: null + landed: + pr: QuantEcon/data-lectures#114 + date: 2026-09-01 + repoints: [] + cutover: null + + private_credit_to_gdp.csv: + pilot: P4 + status: landed + prior_pattern: null + landed: + pr: QuantEcon/data-lectures#114 + date: 2026-09-01 + repoints: [] + cutover: null + + us_business_cycle_monthly.csv: + pilot: P4 + status: landed + prior_pattern: null + landed: + pr: QuantEcon/data-lectures#114 + date: 2026-09-01 + repoints: [] + cutover: null + # Planned waves that have not landed anything here yet. `datasets` names the # files as the audit sees them today, so the dashboard can join the two views. # `title` is the reader-facing milestone name (the dashboard is read by people diff --git a/scripts/snapshots.py b/scripts/snapshots.py index 9e98cd1..1004670 100644 --- a/scripts/snapshots.py +++ b/scripts/snapshots.py @@ -215,8 +215,18 @@ def cmd_due(args) -> int: return 0 +def _load_summary(path: str, dataset: str) -> dict: + """A builder writes one summary (a dict) or, when it produces a set of + files, one per file (a list); either way, the entry for `dataset`.""" + loaded = json.loads(pathlib.Path(path).read_text()) + if isinstance(loaded, list): + matches = [s for s in loaded if s.get("dataset") == dataset] + return matches[0] if matches else {"dataset": None} + return loaded + + def cmd_stamp(args) -> int: - summary = json.loads(pathlib.Path(args.summary).read_text()) + summary = _load_summary(args.summary, args.dataset) dataset = args.dataset if summary.get("dataset") != dataset: # Not an assert: `python -O` would drop it, and this is the guard that @@ -259,7 +269,8 @@ def cmd_stamp(args) -> int: "date": _iso(up["date"]) == today, "against": up["against"] == summary["builder"], "delta dropped": not any(k in up for k in ("delta_kind", "delta", "delta_evidence", "register")), - "date_range.end": m["schema"]["date_range"]["end"] == end, + # YAML reads a date-shaped scalar back as a date object; compare as text. + "date_range.end": _iso(m["schema"]["date_range"]["end"]) == str(end), } failed = [k for k, ok in checks.items() if not ok] if failed: @@ -271,7 +282,7 @@ def cmd_stamp(args) -> int: def cmd_pr_body(args) -> int: - summary = json.loads(pathlib.Path(args.summary).read_text()) + summary = _load_summary(args.summary, args.dataset) dataset = args.dataset m = load_manifests()[dataset] ov = summary.get("overlap") or {} From 4a5b6ddb4b2a308d2a31a7e14a997b0f3e1c6044 Mon Sep 17 00:00:00 2001 From: Matt McKay Date: Tue, 1 Sep 2026 15:57:55 +1000 Subject: [PATCH 2/2] Copilot review on #114: shared summary guard, URL-encoded query, no cast before validate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snapshots.py: the dataset-mismatch guard moves into _load_summary(), so pr-body refuses a foreign or missing summary exactly as stamp does instead of raising KeyError. _fred.py: the query string goes through urllib.parse.urlencode. business_cycle_fred.py: pre_process() no longer casts USREC — a `.` from FRED would have raised ValueError with exit 1 (a retry) instead of a ValidationError with exit 2 (a human); validate() now asserts USREC is complete and 0/1, and run() casts after validation. Output bytes unchanged. Co-Authored-By: Claude Fable 5 --- builders/_fred.py | 3 ++- builders/business_cycle_fred.py | 6 +++++- scripts/snapshots.py | 25 +++++++++++++------------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/builders/_fred.py b/builders/_fred.py index d7b8498..e07e13a 100644 --- a/builders/_fred.py +++ b/builders/_fred.py @@ -30,6 +30,7 @@ their own directory as sys.path[0]). """ import io +import urllib.parse import urllib.request import pandas as pd @@ -43,7 +44,7 @@ def __init__(self, user_agent='qeld-builder', timeout=60): self.timeout = timeout def _get(self, params): - query = '&'.join(f'{k}={v}' for k, v in params.items() if v is not None) + query = urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}) request = urllib.request.Request(f'{FREDGRAPH}?{query}', headers={'User-Agent': self.user_agent}) with urllib.request.urlopen(request, timeout=self.timeout) as response: diff --git a/builders/business_cycle_fred.py b/builders/business_cycle_fred.py index cdac113..218d78d 100644 --- a/builders/business_cycle_fred.py +++ b/builders/business_cycle_fred.py @@ -89,9 +89,11 @@ def fetch(): def pre_process(frame): + # Pure: no casts that can raise. USREC stays float here (a `.` from FRED + # would be NaN) so that validate() is what rejects a hole in it, with + # exit code 2, rather than an astype() raising ValueError with exit 1. frame = frame.loc[START:].copy() frame.index.name = 'DATE' - frame['USREC'] = frame['USREC'].astype('int64') return frame[COLUMNS] @@ -124,6 +126,7 @@ def validate(frame, previous=None): _check(s.loc[last + pd.offsets.MonthBegin(1):].isnull().all(), f'{col}: observed after {last.date()}') lo, hi = BANDS[col] _check(s.dropna().between(lo, hi).all(), f'{col}: out of band [{lo}, {hi}]') + _check(frame['USREC'].notnull().all(), 'USREC has a missing month') _check(set(frame['USREC'].unique()) <= {0, 1}, 'USREC not 0/1') summary = { @@ -172,6 +175,7 @@ def run(out_dir=None, summary_json=None): if os.path.exists(previous_path) else None) frame = pre_process(fetch()) summary = validate(frame, previous) + frame['USREC'] = frame['USREC'].astype('int64') # safe: validated complete and 0/1 if summary_json: _atomic_write(summary_json, json.dumps(summary, indent=1) + '\n') os.makedirs(data_dir, exist_ok=True) diff --git a/scripts/snapshots.py b/scripts/snapshots.py index 1004670..1f42cda 100644 --- a/scripts/snapshots.py +++ b/scripts/snapshots.py @@ -217,23 +217,24 @@ def cmd_due(args) -> int: def _load_summary(path: str, dataset: str) -> dict: """A builder writes one summary (a dict) or, when it produces a set of - files, one per file (a list); either way, the entry for `dataset`.""" + files, one per file (a list); either way, the entry for `dataset` — and + refuse anything else. Not an assert: `python -O` would drop it, and this + is the guard that stops one builder's summary from stamping, or + describing, another dataset.""" loaded = json.loads(pathlib.Path(path).read_text()) - if isinstance(loaded, list): - matches = [s for s in loaded if s.get("dataset") == dataset] - return matches[0] if matches else {"dataset": None} - return loaded + entries = loaded if isinstance(loaded, list) else [loaded] + matches = [s for s in entries if isinstance(s, dict) and s.get("dataset") == dataset] + if not matches: + found = sorted(str(s.get("dataset")) for s in entries if isinstance(s, dict)) + print(f"::error::{dataset}: no summary for it in {path} (found {found}) — refusing", + file=sys.stderr) + raise SystemExit(1) + return matches[0] def cmd_stamp(args) -> int: - summary = _load_summary(args.summary, args.dataset) + summary = _load_summary(args.summary, args.dataset) # refuses a mismatch dataset = args.dataset - if summary.get("dataset") != dataset: - # Not an assert: `python -O` would drop it, and this is the guard that - # stops one builder's summary from stamping another dataset's manifest. - print(f"::error::{dataset}: summary is for {summary.get('dataset')!r}, " - f"not {dataset!r} — refusing to stamp", file=sys.stderr) - return 1 data_path = LECTURES / dataset manifest_path = LECTURES / f"{dataset}.yml" today = dt.date.today().isoformat()